2024-11-17 20:33:17,387 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca
2024-11-17 20:33:17,398 main DEBUG Took 0.009465 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging
2024-11-17 20:33:17,399 main DEBUG PluginManager 'Core' found 129 plugins
2024-11-17 20:33:17,399 main DEBUG PluginManager 'Level' found 0 plugins
2024-11-17 20:33:17,400 main DEBUG PluginManager 'Lookup' found 16 plugins
2024-11-17 20:33:17,401 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-17 20:33:17,407 main DEBUG PluginManager 'TypeConverter' found 26 plugins
2024-11-17 20:33:17,422 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-17 20:33:17,424 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-17 20:33:17,425 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-17 20:33:17,425 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-17 20:33:17,426 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-17 20:33:17,427 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-17 20:33:17,428 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-17 20:33:17,429 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-17 20:33:17,430 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-17 20:33:17,430 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-17 20:33:17,431 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-17 20:33:17,432 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-17 20:33:17,433 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-17 20:33:17,433 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-17 20:33:17,434 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-17 20:33:17,434 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-17 20:33:17,435 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-17 20:33:17,435 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-17 20:33:17,436 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-17 20:33:17,437 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-17 20:33:17,437 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-17 20:33:17,438 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-17 20:33:17,438 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-17 20:33:17,439 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-17 20:33:17,440 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-17 20:33:17,440 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger].
2024-11-17 20:33:17,442 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-17 20:33:17,444 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin].
2024-11-17 20:33:17,446 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root})
2024-11-17 20:33:17,447 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout].
2024-11-17 20:33:17,448 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null")
2024-11-17 20:33:17,449 main DEBUG PluginManager 'Converter' found 47 plugins
2024-11-17 20:33:17,460 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender].
2024-11-17 20:33:17,464 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={})
2024-11-17 20:33:17,466 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR
2024-11-17 20:33:17,467 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin].
2024-11-17 20:33:17,467 main DEBUG createAppenders(={Console})
2024-11-17 20:33:17,468 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca initialized
2024-11-17 20:33:17,469 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca
2024-11-17 20:33:17,469 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca OK.
2024-11-17 20:33:17,470 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1
2024-11-17 20:33:17,470 main DEBUG OutputStream closed
2024-11-17 20:33:17,471 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true
2024-11-17 20:33:17,471 main DEBUG Appender DefaultConsole-1 stopped with status true
2024-11-17 20:33:17,471 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@6404f418 OK
2024-11-17 20:33:17,542 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6
2024-11-17 20:33:17,544 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger
2024-11-17 20:33:17,545 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector
2024-11-17 20:33:17,546 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=
2024-11-17 20:33:17,546 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory
2024-11-17 20:33:17,547 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter
2024-11-17 20:33:17,547 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper
2024-11-17 20:33:17,547 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j
2024-11-17 20:33:17,547 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl
2024-11-17 20:33:17,548 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans
2024-11-17 20:33:17,548 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase
2024-11-17 20:33:17,548 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop
2024-11-17 20:33:17,549 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers
2024-11-17 20:33:17,549 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices
2024-11-17 20:33:17,549 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig
2024-11-17 20:33:17,549 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel
2024-11-17 20:33:17,550 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore
2024-11-17 20:33:17,550 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console
2024-11-17 20:33:17,552 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-11-17 20:33:17,553 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@6dab9b6d) with optional ClassLoader: null
2024-11-17 20:33:17,553 main DEBUG Shutdown hook enabled. Registering a new one.
2024-11-17 20:33:17,554 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@6dab9b6d] started OK.
2024-11-17T20:33:17,775 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d5cf336-d7e3-9af4-11bd-2d3b3572aece
2024-11-17 20:33:17,777 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED)
2024-11-17 20:33:17,778 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-11-17T20:33:17,786 INFO  [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestLogRolling timeout: 13 mins
2024-11-17T20:33:17,819 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testSlowSyncLogRolling Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=251, ProcessCount=11, AvailableMemoryMB=8887
2024-11-17T20:33:17,822 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-17T20:33:17,842 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d5cf336-d7e3-9af4-11bd-2d3b3572aece/cluster_736ee10e-3507-4deb-8e28-689799b59293, deleteOnExit=true
2024-11-17T20:33:17,842 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-17T20:33:17,844 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d5cf336-d7e3-9af4-11bd-2d3b3572aece/test.cache.data in system properties and HBase conf
2024-11-17T20:33:17,845 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d5cf336-d7e3-9af4-11bd-2d3b3572aece/hadoop.tmp.dir in system properties and HBase conf
2024-11-17T20:33:17,846 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d5cf336-d7e3-9af4-11bd-2d3b3572aece/hadoop.log.dir in system properties and HBase conf
2024-11-17T20:33:17,847 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d5cf336-d7e3-9af4-11bd-2d3b3572aece/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-17T20:33:17,847 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d5cf336-d7e3-9af4-11bd-2d3b3572aece/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-17T20:33:17,848 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-17T20:33:17,948 WARN  [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
2024-11-17T20:33:18,036 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-17T20:33:18,040 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d5cf336-d7e3-9af4-11bd-2d3b3572aece/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-17T20:33:18,041 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d5cf336-d7e3-9af4-11bd-2d3b3572aece/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-17T20:33:18,042 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d5cf336-d7e3-9af4-11bd-2d3b3572aece/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-17T20:33:18,042 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d5cf336-d7e3-9af4-11bd-2d3b3572aece/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-17T20:33:18,043 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d5cf336-d7e3-9af4-11bd-2d3b3572aece/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-17T20:33:18,044 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d5cf336-d7e3-9af4-11bd-2d3b3572aece/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-17T20:33:18,044 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d5cf336-d7e3-9af4-11bd-2d3b3572aece/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-17T20:33:18,045 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d5cf336-d7e3-9af4-11bd-2d3b3572aece/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-17T20:33:18,045 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d5cf336-d7e3-9af4-11bd-2d3b3572aece/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-17T20:33:18,046 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d5cf336-d7e3-9af4-11bd-2d3b3572aece/nfs.dump.dir in system properties and HBase conf
2024-11-17T20:33:18,047 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d5cf336-d7e3-9af4-11bd-2d3b3572aece/java.io.tmpdir in system properties and HBase conf
2024-11-17T20:33:18,047 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d5cf336-d7e3-9af4-11bd-2d3b3572aece/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-17T20:33:18,048 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d5cf336-d7e3-9af4-11bd-2d3b3572aece/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-17T20:33:18,049 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d5cf336-d7e3-9af4-11bd-2d3b3572aece/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-17T20:33:18,463 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-17T20:33:19,097 WARN  [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
2024-11-17T20:33:19,163 INFO  [Time-limited test {}] log.Log(170): Logging initialized @2448ms to org.eclipse.jetty.util.log.Slf4jLog
2024-11-17T20:33:19,227 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-17T20:33:19,287 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-17T20:33:19,308 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-17T20:33:19,308 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-17T20:33:19,309 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-17T20:33:19,321 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-17T20:33:19,324 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@ddc8467{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d5cf336-d7e3-9af4-11bd-2d3b3572aece/hadoop.log.dir/,AVAILABLE}
2024-11-17T20:33:19,325 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@70be1389{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-17T20:33:19,493 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@735fa16a{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d5cf336-d7e3-9af4-11bd-2d3b3572aece/java.io.tmpdir/jetty-localhost-43879-hadoop-hdfs-3_4_1-tests_jar-_-any-15028178725553746853/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-17T20:33:19,500 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6c26a5a3{HTTP/1.1, (http/1.1)}{localhost:43879}
2024-11-17T20:33:19,500 INFO  [Time-limited test {}] server.Server(415): Started @2786ms
2024-11-17T20:33:19,531 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-17T20:33:20,086 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-17T20:33:20,094 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-17T20:33:20,095 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-17T20:33:20,095 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-17T20:33:20,095 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-17T20:33:20,096 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@28778f0f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d5cf336-d7e3-9af4-11bd-2d3b3572aece/hadoop.log.dir/,AVAILABLE}
2024-11-17T20:33:20,097 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@371e191c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-17T20:33:20,196 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7b07d1ba{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d5cf336-d7e3-9af4-11bd-2d3b3572aece/java.io.tmpdir/jetty-localhost-33589-hadoop-hdfs-3_4_1-tests_jar-_-any-18264831567412839690/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T20:33:20,196 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@43e0a762{HTTP/1.1, (http/1.1)}{localhost:33589}
2024-11-17T20:33:20,196 INFO  [Time-limited test {}] server.Server(415): Started @3482ms
2024-11-17T20:33:20,247 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-17T20:33:20,355 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-17T20:33:20,362 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-17T20:33:20,364 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-17T20:33:20,365 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-17T20:33:20,365 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-17T20:33:20,367 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@11effdcd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d5cf336-d7e3-9af4-11bd-2d3b3572aece/hadoop.log.dir/,AVAILABLE}
2024-11-17T20:33:20,368 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2d48d695{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-17T20:33:20,494 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1bf97579{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d5cf336-d7e3-9af4-11bd-2d3b3572aece/java.io.tmpdir/jetty-localhost-34769-hadoop-hdfs-3_4_1-tests_jar-_-any-1312741918020960700/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T20:33:20,494 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@22b88bcb{HTTP/1.1, (http/1.1)}{localhost:34769}
2024-11-17T20:33:20,495 INFO  [Time-limited test {}] server.Server(415): Started @3781ms
2024-11-17T20:33:20,497 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-17T20:33:22,266 WARN  [Thread-101 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d5cf336-d7e3-9af4-11bd-2d3b3572aece/cluster_736ee10e-3507-4deb-8e28-689799b59293/data/data4/current/BP-1663635380-172.17.0.2-1731875598538/current, will proceed with Du for space computation calculation, 
2024-11-17T20:33:22,266 WARN  [Thread-98 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d5cf336-d7e3-9af4-11bd-2d3b3572aece/cluster_736ee10e-3507-4deb-8e28-689799b59293/data/data1/current/BP-1663635380-172.17.0.2-1731875598538/current, will proceed with Du for space computation calculation, 
2024-11-17T20:33:22,266 WARN  [Thread-100 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d5cf336-d7e3-9af4-11bd-2d3b3572aece/cluster_736ee10e-3507-4deb-8e28-689799b59293/data/data3/current/BP-1663635380-172.17.0.2-1731875598538/current, will proceed with Du for space computation calculation, 
2024-11-17T20:33:22,266 WARN  [Thread-99 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d5cf336-d7e3-9af4-11bd-2d3b3572aece/cluster_736ee10e-3507-4deb-8e28-689799b59293/data/data2/current/BP-1663635380-172.17.0.2-1731875598538/current, will proceed with Du for space computation calculation, 
2024-11-17T20:33:22,305 WARN  [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-17T20:33:22,307 WARN  [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-17T20:33:22,355 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3a79530f851540ff with lease ID 0xca70c712ec91cfdd: Processing first storage report for DS-5f2c1e58-7b56-4468-be16-067721c1967c from datanode DatanodeRegistration(127.0.0.1:46521, datanodeUuid=93d16be3-7334-46a2-96dc-9dfa33e1fa4c, infoPort=34187, infoSecurePort=0, ipcPort=37167, storageInfo=lv=-57;cid=testClusterID;nsid=1417548587;c=1731875598538)
2024-11-17T20:33:22,356 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3a79530f851540ff with lease ID 0xca70c712ec91cfdd: from storage DS-5f2c1e58-7b56-4468-be16-067721c1967c node DatanodeRegistration(127.0.0.1:46521, datanodeUuid=93d16be3-7334-46a2-96dc-9dfa33e1fa4c, infoPort=34187, infoSecurePort=0, ipcPort=37167, storageInfo=lv=-57;cid=testClusterID;nsid=1417548587;c=1731875598538), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0
2024-11-17T20:33:22,357 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xda7a25a49e098864 with lease ID 0xca70c712ec91cfdc: Processing first storage report for DS-53159149-da7e-4ad8-9376-eaab27d68ac7 from datanode DatanodeRegistration(127.0.0.1:34923, datanodeUuid=d62433fe-f6f4-4dd0-865c-90eff6d6a8d3, infoPort=33193, infoSecurePort=0, ipcPort=38637, storageInfo=lv=-57;cid=testClusterID;nsid=1417548587;c=1731875598538)
2024-11-17T20:33:22,357 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xda7a25a49e098864 with lease ID 0xca70c712ec91cfdc: from storage DS-53159149-da7e-4ad8-9376-eaab27d68ac7 node DatanodeRegistration(127.0.0.1:34923, datanodeUuid=d62433fe-f6f4-4dd0-865c-90eff6d6a8d3, infoPort=33193, infoSecurePort=0, ipcPort=38637, storageInfo=lv=-57;cid=testClusterID;nsid=1417548587;c=1731875598538), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-17T20:33:22,357 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3a79530f851540ff with lease ID 0xca70c712ec91cfdd: Processing first storage report for DS-2c2e8ebb-291e-479f-ab8a-fd712a5ea311 from datanode DatanodeRegistration(127.0.0.1:46521, datanodeUuid=93d16be3-7334-46a2-96dc-9dfa33e1fa4c, infoPort=34187, infoSecurePort=0, ipcPort=37167, storageInfo=lv=-57;cid=testClusterID;nsid=1417548587;c=1731875598538)
2024-11-17T20:33:22,357 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3a79530f851540ff with lease ID 0xca70c712ec91cfdd: from storage DS-2c2e8ebb-291e-479f-ab8a-fd712a5ea311 node DatanodeRegistration(127.0.0.1:46521, datanodeUuid=93d16be3-7334-46a2-96dc-9dfa33e1fa4c, infoPort=34187, infoSecurePort=0, ipcPort=37167, storageInfo=lv=-57;cid=testClusterID;nsid=1417548587;c=1731875598538), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-17T20:33:22,358 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xda7a25a49e098864 with lease ID 0xca70c712ec91cfdc: Processing first storage report for DS-756758ab-af49-4fa9-8f27-f0673b84d6fb from datanode DatanodeRegistration(127.0.0.1:34923, datanodeUuid=d62433fe-f6f4-4dd0-865c-90eff6d6a8d3, infoPort=33193, infoSecurePort=0, ipcPort=38637, storageInfo=lv=-57;cid=testClusterID;nsid=1417548587;c=1731875598538)
2024-11-17T20:33:22,358 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xda7a25a49e098864 with lease ID 0xca70c712ec91cfdc: from storage DS-756758ab-af49-4fa9-8f27-f0673b84d6fb node DatanodeRegistration(127.0.0.1:34923, datanodeUuid=d62433fe-f6f4-4dd0-865c-90eff6d6a8d3, infoPort=33193, infoSecurePort=0, ipcPort=38637, storageInfo=lv=-57;cid=testClusterID;nsid=1417548587;c=1731875598538), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-17T20:33:22,431 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d5cf336-d7e3-9af4-11bd-2d3b3572aece
2024-11-17T20:33:22,491 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d5cf336-d7e3-9af4-11bd-2d3b3572aece/cluster_736ee10e-3507-4deb-8e28-689799b59293/zookeeper_0, clientPort=51492, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d5cf336-d7e3-9af4-11bd-2d3b3572aece/cluster_736ee10e-3507-4deb-8e28-689799b59293/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d5cf336-d7e3-9af4-11bd-2d3b3572aece/cluster_736ee10e-3507-4deb-8e28-689799b59293/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-17T20:33:22,499 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=51492
2024-11-17T20:33:22,509 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T20:33:22,511 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T20:33:22,779 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46521 is added to blk_1073741825_1001 (size=7)
2024-11-17T20:33:22,781 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741825_1001 (size=7)
2024-11-17T20:33:23,183 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d with version=8
2024-11-17T20:33:23,183 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/hbase-staging
2024-11-17T20:33:23,280 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16
2024-11-17T20:33:23,570 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/0b0b0ae8de03:0 server-side Connection retries=45
2024-11-17T20:33:23,580 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-17T20:33:23,581 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-17T20:33:23,586 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-17T20:33:23,586 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-17T20:33:23,587 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-17T20:33:23,731 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-17T20:33:23,808 INFO  [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl
2024-11-17T20:33:23,821 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout
2024-11-17T20:33:23,826 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-17T20:33:23,857 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 100490 (auto-detected)
2024-11-17T20:33:23,859 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected)
2024-11-17T20:33:23,884 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35281
2024-11-17T20:33:23,916 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:35281 connecting to ZooKeeper ensemble=127.0.0.1:51492
2024-11-17T20:33:24,067 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:352810x0, quorum=127.0.0.1:51492, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-17T20:33:24,070 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:35281-0x1014a807bd70000 connected
2024-11-17T20:33:24,178 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T20:33:24,184 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T20:33:24,196 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35281-0x1014a807bd70000, quorum=127.0.0.1:51492, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-17T20:33:24,200 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d, hbase.cluster.distributed=false
2024-11-17T20:33:24,223 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35281-0x1014a807bd70000, quorum=127.0.0.1:51492, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-17T20:33:24,228 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35281
2024-11-17T20:33:24,228 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35281
2024-11-17T20:33:24,229 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35281
2024-11-17T20:33:24,229 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35281
2024-11-17T20:33:24,230 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35281
2024-11-17T20:33:24,326 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/0b0b0ae8de03:0 server-side Connection retries=45
2024-11-17T20:33:24,328 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-17T20:33:24,328 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-17T20:33:24,328 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-17T20:33:24,328 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-17T20:33:24,328 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-17T20:33:24,331 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-17T20:33:24,333 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-17T20:33:24,334 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34589
2024-11-17T20:33:24,336 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34589 connecting to ZooKeeper ensemble=127.0.0.1:51492
2024-11-17T20:33:24,337 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T20:33:24,342 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T20:33:24,360 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:345890x0, quorum=127.0.0.1:51492, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-17T20:33:24,361 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:345890x0, quorum=127.0.0.1:51492, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-17T20:33:24,361 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34589-0x1014a807bd70001 connected
2024-11-17T20:33:24,365 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-17T20:33:24,371 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-17T20:33:24,374 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34589-0x1014a807bd70001, quorum=127.0.0.1:51492, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-17T20:33:24,379 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34589-0x1014a807bd70001, quorum=127.0.0.1:51492, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-17T20:33:24,380 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34589
2024-11-17T20:33:24,381 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34589
2024-11-17T20:33:24,382 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34589
2024-11-17T20:33:24,383 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34589
2024-11-17T20:33:24,383 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34589
2024-11-17T20:33:24,399 DEBUG [M:0;0b0b0ae8de03:35281 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;0b0b0ae8de03:35281
2024-11-17T20:33:24,400 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/0b0b0ae8de03,35281,1731875603390
2024-11-17T20:33:24,417 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35281-0x1014a807bd70000, quorum=127.0.0.1:51492, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-17T20:33:24,417 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34589-0x1014a807bd70001, quorum=127.0.0.1:51492, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-17T20:33:24,419 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35281-0x1014a807bd70000, quorum=127.0.0.1:51492, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/0b0b0ae8de03,35281,1731875603390
2024-11-17T20:33:24,448 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34589-0x1014a807bd70001, quorum=127.0.0.1:51492, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-17T20:33:24,448 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35281-0x1014a807bd70000, quorum=127.0.0.1:51492, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:33:24,448 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34589-0x1014a807bd70001, quorum=127.0.0.1:51492, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:33:24,449 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35281-0x1014a807bd70000, quorum=127.0.0.1:51492, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-17T20:33:24,450 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/0b0b0ae8de03,35281,1731875603390 from backup master directory
2024-11-17T20:33:24,458 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34589-0x1014a807bd70001, quorum=127.0.0.1:51492, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-17T20:33:24,458 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35281-0x1014a807bd70000, quorum=127.0.0.1:51492, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/0b0b0ae8de03,35281,1731875603390
2024-11-17T20:33:24,459 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35281-0x1014a807bd70000, quorum=127.0.0.1:51492, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-17T20:33:24,459 WARN  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-17T20:33:24,459 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=0b0b0ae8de03,35281,1731875603390
2024-11-17T20:33:24,461 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0
2024-11-17T20:33:24,462 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0
2024-11-17T20:33:24,521 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/hbase.id] with ID: 900a8cee-cf87-4610-9c36-47ef82d3dd56
2024-11-17T20:33:24,522 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/.tmp/hbase.id
2024-11-17T20:33:24,533 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46521 is added to blk_1073741826_1002 (size=42)
2024-11-17T20:33:24,533 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741826_1002 (size=42)
2024-11-17T20:33:24,534 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/.tmp/hbase.id]:[hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/hbase.id]
2024-11-17T20:33:24,577 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T20:33:24,581 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-17T20:33:24,600 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms.
2024-11-17T20:33:24,613 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35281-0x1014a807bd70000, quorum=127.0.0.1:51492, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:33:24,613 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34589-0x1014a807bd70001, quorum=127.0.0.1:51492, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:33:24,631 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46521 is added to blk_1073741827_1003 (size=196)
2024-11-17T20:33:24,632 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741827_1003 (size=196)
2024-11-17T20:33:24,645 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-17T20:33:24,647 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-17T20:33:24,652 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-17T20:33:24,680 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741828_1004 (size=1189)
2024-11-17T20:33:24,680 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46521 is added to blk_1073741828_1004 (size=1189)
2024-11-17T20:33:24,697 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/MasterData/data/master/store
2024-11-17T20:33:24,716 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741829_1005 (size=34)
2024-11-17T20:33:24,717 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46521 is added to blk_1073741829_1005 (size=34)
2024-11-17T20:33:24,721 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure.
2024-11-17T20:33:24,724 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T20:33:24,725 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-17T20:33:24,725 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T20:33:24,725 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T20:33:24,727 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-17T20:33:24,727 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T20:33:24,727 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T20:33:24,728 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731875604725Disabling compacts and flushes for region at 1731875604725Disabling writes for close at 1731875604727 (+2 ms)Writing region close event to WAL at 1731875604727Closed at 1731875604727
2024-11-17T20:33:24,730 WARN  [master/0b0b0ae8de03:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/MasterData/data/master/store/.initializing
2024-11-17T20:33:24,730 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/MasterData/WALs/0b0b0ae8de03,35281,1731875603390
2024-11-17T20:33:24,750 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0b0b0ae8de03%2C35281%2C1731875603390, suffix=, logDir=hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/MasterData/WALs/0b0b0ae8de03,35281,1731875603390, archiveDir=hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/MasterData/oldWALs, maxLogs=10
2024-11-17T20:33:24,757 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0b0b0ae8de03%2C35281%2C1731875603390.1731875604754
2024-11-17T20:33:24,777 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/MasterData/WALs/0b0b0ae8de03,35281,1731875603390/0b0b0ae8de03%2C35281%2C1731875603390.1731875604754
2024-11-17T20:33:24,784 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34187:34187),(127.0.0.1/127.0.0.1:33193:33193)]
2024-11-17T20:33:24,785 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-17T20:33:24,786 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T20:33:24,789 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T20:33:24,790 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T20:33:24,829 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-17T20:33:24,850 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-17T20:33:24,853 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:33:24,855 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T20:33:24,856 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-17T20:33:24,859 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-17T20:33:24,859 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:33:24,860 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-17T20:33:24,860 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-17T20:33:24,863 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-17T20:33:24,863 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:33:24,864 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-17T20:33:24,865 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-17T20:33:24,867 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-17T20:33:24,867 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:33:24,868 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-17T20:33:24,869 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T20:33:24,873 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-17T20:33:24,874 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-17T20:33:24,879 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T20:33:24,879 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T20:33:24,882 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-17T20:33:24,885 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T20:33:24,889 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-17T20:33:24,891 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=856780, jitterRate=0.08945272862911224}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-17T20:33:24,898 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1731875604807Initializing all the Stores at 1731875604809 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731875604810 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731875604810Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731875604811 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731875604811Cleaning up temporary data from old regions at 1731875604880 (+69 ms)Region opened successfully at 1731875604898 (+18 ms)
2024-11-17T20:33:24,899 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-17T20:33:24,928 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d7f756f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0b0b0ae8de03/172.17.0.2:0
2024-11-17T20:33:24,954 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-17T20:33:24,963 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-17T20:33:24,964 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-17T20:33:24,966 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-17T20:33:24,967 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec
2024-11-17T20:33:24,971 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec
2024-11-17T20:33:24,972 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-17T20:33:24,995 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-17T20:33:25,003 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35281-0x1014a807bd70000, quorum=127.0.0.1:51492, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-17T20:33:25,016 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-17T20:33:25,019 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-17T20:33:25,020 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35281-0x1014a807bd70000, quorum=127.0.0.1:51492, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-17T20:33:25,108 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-17T20:33:25,112 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-17T20:33:25,119 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35281-0x1014a807bd70000, quorum=127.0.0.1:51492, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-17T20:33:25,132 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-17T20:33:25,133 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35281-0x1014a807bd70000, quorum=127.0.0.1:51492, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-17T20:33:25,142 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-17T20:33:25,162 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35281-0x1014a807bd70000, quorum=127.0.0.1:51492, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-17T20:33:25,174 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-17T20:33:25,185 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35281-0x1014a807bd70000, quorum=127.0.0.1:51492, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-17T20:33:25,185 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34589-0x1014a807bd70001, quorum=127.0.0.1:51492, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-17T20:33:25,185 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35281-0x1014a807bd70000, quorum=127.0.0.1:51492, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:33:25,185 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34589-0x1014a807bd70001, quorum=127.0.0.1:51492, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:33:25,189 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=0b0b0ae8de03,35281,1731875603390, sessionid=0x1014a807bd70000, setting cluster-up flag (Was=false)
2024-11-17T20:33:25,223 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35281-0x1014a807bd70000, quorum=127.0.0.1:51492, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:33:25,223 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34589-0x1014a807bd70001, quorum=127.0.0.1:51492, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:33:25,259 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-17T20:33:25,262 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0b0b0ae8de03,35281,1731875603390
2024-11-17T20:33:25,286 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34589-0x1014a807bd70001, quorum=127.0.0.1:51492, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:33:25,286 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35281-0x1014a807bd70000, quorum=127.0.0.1:51492, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:33:25,322 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-17T20:33:25,326 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0b0b0ae8de03,35281,1731875603390
2024-11-17T20:33:25,337 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-17T20:33:25,389 INFO  [RS:0;0b0b0ae8de03:34589 {}] regionserver.HRegionServer(746): ClusterId : 900a8cee-cf87-4610-9c36-47ef82d3dd56
2024-11-17T20:33:25,392 DEBUG [RS:0;0b0b0ae8de03:34589 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-17T20:33:25,404 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-17T20:33:25,408 DEBUG [RS:0;0b0b0ae8de03:34589 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-17T20:33:25,408 DEBUG [RS:0;0b0b0ae8de03:34589 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-17T20:33:25,415 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-17T20:33:25,418 DEBUG [RS:0;0b0b0ae8de03:34589 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-17T20:33:25,418 DEBUG [RS:0;0b0b0ae8de03:34589 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35d8520b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0b0b0ae8de03/172.17.0.2:0
2024-11-17T20:33:25,423 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-17T20:33:25,433 DEBUG [RS:0;0b0b0ae8de03:34589 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;0b0b0ae8de03:34589
2024-11-17T20:33:25,430 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 0b0b0ae8de03,35281,1731875603390
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-17T20:33:25,435 INFO  [RS:0;0b0b0ae8de03:34589 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-17T20:33:25,436 INFO  [RS:0;0b0b0ae8de03:34589 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-17T20:33:25,436 DEBUG [RS:0;0b0b0ae8de03:34589 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-17T20:33:25,438 INFO  [RS:0;0b0b0ae8de03:34589 {}] regionserver.HRegionServer(2659): reportForDuty to master=0b0b0ae8de03,35281,1731875603390 with port=34589, startcode=1731875604293
2024-11-17T20:33:25,439 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/0b0b0ae8de03:0, corePoolSize=5, maxPoolSize=5
2024-11-17T20:33:25,439 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/0b0b0ae8de03:0, corePoolSize=5, maxPoolSize=5
2024-11-17T20:33:25,439 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/0b0b0ae8de03:0, corePoolSize=5, maxPoolSize=5
2024-11-17T20:33:25,439 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/0b0b0ae8de03:0, corePoolSize=5, maxPoolSize=5
2024-11-17T20:33:25,440 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/0b0b0ae8de03:0, corePoolSize=10, maxPoolSize=10
2024-11-17T20:33:25,440 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:33:25,440 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/0b0b0ae8de03:0, corePoolSize=2, maxPoolSize=2
2024-11-17T20:33:25,440 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:33:25,445 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-17T20:33:25,445 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-17T20:33:25,447 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731875635447
2024-11-17T20:33:25,449 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-17T20:33:25,449 DEBUG [RS:0;0b0b0ae8de03:34589 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-17T20:33:25,449 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-17T20:33:25,451 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:33:25,452 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-17T20:33:25,452 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-17T20:33:25,453 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-17T20:33:25,453 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-17T20:33:25,453 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-17T20:33:25,454 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-17T20:33:25,460 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-17T20:33:25,461 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-17T20:33:25,461 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-17T20:33:25,463 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-17T20:33:25,463 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-17T20:33:25,465 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/0b0b0ae8de03:0:becomeActiveMaster-HFileCleaner.large.0-1731875605465,5,FailOnTimeoutGroup]
2024-11-17T20:33:25,466 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/0b0b0ae8de03:0:becomeActiveMaster-HFileCleaner.small.0-1731875605466,5,FailOnTimeoutGroup]
2024-11-17T20:33:25,466 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-17T20:33:25,467 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-17T20:33:25,467 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741831_1007 (size=1321)
2024-11-17T20:33:25,467 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46521 is added to blk_1073741831_1007 (size=1321)
2024-11-17T20:33:25,468 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-17T20:33:25,469 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-17T20:33:25,469 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-17T20:33:25,469 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d
2024-11-17T20:33:25,480 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46521 is added to blk_1073741832_1008 (size=32)
2024-11-17T20:33:25,480 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741832_1008 (size=32)
2024-11-17T20:33:25,482 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T20:33:25,485 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-17T20:33:25,488 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-17T20:33:25,488 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:33:25,489 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T20:33:25,489 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-17T20:33:25,493 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-17T20:33:25,493 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:33:25,494 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T20:33:25,494 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-17T20:33:25,497 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-17T20:33:25,497 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:33:25,499 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T20:33:25,499 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-17T20:33:25,502 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-17T20:33:25,502 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:33:25,503 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T20:33:25,503 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-17T20:33:25,505 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/data/hbase/meta/1588230740
2024-11-17T20:33:25,506 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/data/hbase/meta/1588230740
2024-11-17T20:33:25,509 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-17T20:33:25,509 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-17T20:33:25,510 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-17T20:33:25,512 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-17T20:33:25,516 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-17T20:33:25,518 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=751300, jitterRate=-0.0446736216545105}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-17T20:33:25,519 INFO  [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45223, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService
2024-11-17T20:33:25,521 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1731875605482Initializing all the Stores at 1731875605484 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731875605484Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731875605485 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731875605485Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731875605485Cleaning up temporary data from old regions at 1731875605509 (+24 ms)Region opened successfully at 1731875605521 (+12 ms)
2024-11-17T20:33:25,523 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-17T20:33:25,523 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-17T20:33:25,523 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-17T20:33:25,523 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-17T20:33:25,523 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-17T20:33:25,524 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-17T20:33:25,524 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731875605523Disabling compacts and flushes for region at 1731875605523Disabling writes for close at 1731875605523Writing region close event to WAL at 1731875605524 (+1 ms)Closed at 1731875605524
2024-11-17T20:33:25,525 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35281 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 0b0b0ae8de03,34589,1731875604293
2024-11-17T20:33:25,527 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35281 {}] master.ServerManager(517): Registering regionserver=0b0b0ae8de03,34589,1731875604293
2024-11-17T20:33:25,527 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-17T20:33:25,527 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-17T20:33:25,533 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-17T20:33:25,540 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-17T20:33:25,540 DEBUG [RS:0;0b0b0ae8de03:34589 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d
2024-11-17T20:33:25,541 DEBUG [RS:0;0b0b0ae8de03:34589 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35585
2024-11-17T20:33:25,541 DEBUG [RS:0;0b0b0ae8de03:34589 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-17T20:33:25,543 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-17T20:33:25,553 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35281-0x1014a807bd70000, quorum=127.0.0.1:51492, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-17T20:33:25,554 DEBUG [RS:0;0b0b0ae8de03:34589 {}] zookeeper.ZKUtil(111): regionserver:34589-0x1014a807bd70001, quorum=127.0.0.1:51492, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/0b0b0ae8de03,34589,1731875604293
2024-11-17T20:33:25,554 WARN  [RS:0;0b0b0ae8de03:34589 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-17T20:33:25,554 INFO  [RS:0;0b0b0ae8de03:34589 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-17T20:33:25,554 DEBUG [RS:0;0b0b0ae8de03:34589 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/WALs/0b0b0ae8de03,34589,1731875604293
2024-11-17T20:33:25,556 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [0b0b0ae8de03,34589,1731875604293]
2024-11-17T20:33:25,577 INFO  [RS:0;0b0b0ae8de03:34589 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-17T20:33:25,591 INFO  [RS:0;0b0b0ae8de03:34589 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-17T20:33:25,595 INFO  [RS:0;0b0b0ae8de03:34589 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-17T20:33:25,595 INFO  [RS:0;0b0b0ae8de03:34589 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-17T20:33:25,596 INFO  [RS:0;0b0b0ae8de03:34589 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-17T20:33:25,601 INFO  [RS:0;0b0b0ae8de03:34589 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-17T20:33:25,602 INFO  [RS:0;0b0b0ae8de03:34589 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-17T20:33:25,602 DEBUG [RS:0;0b0b0ae8de03:34589 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:33:25,603 DEBUG [RS:0;0b0b0ae8de03:34589 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:33:25,603 DEBUG [RS:0;0b0b0ae8de03:34589 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:33:25,603 DEBUG [RS:0;0b0b0ae8de03:34589 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:33:25,603 DEBUG [RS:0;0b0b0ae8de03:34589 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:33:25,603 DEBUG [RS:0;0b0b0ae8de03:34589 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/0b0b0ae8de03:0, corePoolSize=2, maxPoolSize=2
2024-11-17T20:33:25,603 DEBUG [RS:0;0b0b0ae8de03:34589 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:33:25,604 DEBUG [RS:0;0b0b0ae8de03:34589 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:33:25,604 DEBUG [RS:0;0b0b0ae8de03:34589 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:33:25,604 DEBUG [RS:0;0b0b0ae8de03:34589 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:33:25,604 DEBUG [RS:0;0b0b0ae8de03:34589 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:33:25,604 DEBUG [RS:0;0b0b0ae8de03:34589 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:33:25,604 DEBUG [RS:0;0b0b0ae8de03:34589 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/0b0b0ae8de03:0, corePoolSize=3, maxPoolSize=3
2024-11-17T20:33:25,604 DEBUG [RS:0;0b0b0ae8de03:34589 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0, corePoolSize=3, maxPoolSize=3
2024-11-17T20:33:25,606 INFO  [RS:0;0b0b0ae8de03:34589 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-17T20:33:25,606 INFO  [RS:0;0b0b0ae8de03:34589 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-17T20:33:25,606 INFO  [RS:0;0b0b0ae8de03:34589 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-17T20:33:25,606 INFO  [RS:0;0b0b0ae8de03:34589 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-17T20:33:25,606 INFO  [RS:0;0b0b0ae8de03:34589 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-17T20:33:25,606 INFO  [RS:0;0b0b0ae8de03:34589 {}] hbase.ChoreService(168): Chore ScheduledChore name=0b0b0ae8de03,34589,1731875604293-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-17T20:33:25,623 INFO  [RS:0;0b0b0ae8de03:34589 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-17T20:33:25,624 INFO  [RS:0;0b0b0ae8de03:34589 {}] hbase.ChoreService(168): Chore ScheduledChore name=0b0b0ae8de03,34589,1731875604293-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-17T20:33:25,625 INFO  [RS:0;0b0b0ae8de03:34589 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T20:33:25,625 INFO  [RS:0;0b0b0ae8de03:34589 {}] regionserver.Replication(171): 0b0b0ae8de03,34589,1731875604293 started
2024-11-17T20:33:25,640 INFO  [RS:0;0b0b0ae8de03:34589 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T20:33:25,640 INFO  [RS:0;0b0b0ae8de03:34589 {}] regionserver.HRegionServer(1482): Serving as 0b0b0ae8de03,34589,1731875604293, RpcServer on 0b0b0ae8de03/172.17.0.2:34589, sessionid=0x1014a807bd70001
2024-11-17T20:33:25,641 DEBUG [RS:0;0b0b0ae8de03:34589 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-17T20:33:25,641 DEBUG [RS:0;0b0b0ae8de03:34589 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 0b0b0ae8de03,34589,1731875604293
2024-11-17T20:33:25,641 DEBUG [RS:0;0b0b0ae8de03:34589 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0b0b0ae8de03,34589,1731875604293'
2024-11-17T20:33:25,641 DEBUG [RS:0;0b0b0ae8de03:34589 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-17T20:33:25,642 DEBUG [RS:0;0b0b0ae8de03:34589 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-17T20:33:25,643 DEBUG [RS:0;0b0b0ae8de03:34589 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-17T20:33:25,643 DEBUG [RS:0;0b0b0ae8de03:34589 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-17T20:33:25,643 DEBUG [RS:0;0b0b0ae8de03:34589 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 0b0b0ae8de03,34589,1731875604293
2024-11-17T20:33:25,644 DEBUG [RS:0;0b0b0ae8de03:34589 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0b0b0ae8de03,34589,1731875604293'
2024-11-17T20:33:25,644 DEBUG [RS:0;0b0b0ae8de03:34589 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-17T20:33:25,644 DEBUG [RS:0;0b0b0ae8de03:34589 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-17T20:33:25,645 DEBUG [RS:0;0b0b0ae8de03:34589 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-17T20:33:25,645 INFO  [RS:0;0b0b0ae8de03:34589 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-17T20:33:25,645 INFO  [RS:0;0b0b0ae8de03:34589 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-17T20:33:25,694 WARN  [0b0b0ae8de03:35281 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions.
2024-11-17T20:33:25,758 INFO  [RS:0;0b0b0ae8de03:34589 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0b0b0ae8de03%2C34589%2C1731875604293, suffix=, logDir=hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/WALs/0b0b0ae8de03,34589,1731875604293, archiveDir=hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/oldWALs, maxLogs=32
2024-11-17T20:33:25,763 INFO  [RS:0;0b0b0ae8de03:34589 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0b0b0ae8de03%2C34589%2C1731875604293.1731875605763
2024-11-17T20:33:25,772 INFO  [RS:0;0b0b0ae8de03:34589 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/WALs/0b0b0ae8de03,34589,1731875604293/0b0b0ae8de03%2C34589%2C1731875604293.1731875605763
2024-11-17T20:33:25,774 DEBUG [RS:0;0b0b0ae8de03:34589 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34187:34187),(127.0.0.1/127.0.0.1:33193:33193)]
2024-11-17T20:33:25,946 DEBUG [0b0b0ae8de03:35281 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-17T20:33:25,957 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=0b0b0ae8de03,34589,1731875604293
2024-11-17T20:33:25,963 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0b0b0ae8de03,34589,1731875604293, state=OPENING
2024-11-17T20:33:26,016 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-17T20:33:26,027 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35281-0x1014a807bd70000, quorum=127.0.0.1:51492, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:33:26,027 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34589-0x1014a807bd70001, quorum=127.0.0.1:51492, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:33:26,029 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-17T20:33:26,030 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-17T20:33:26,033 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-17T20:33:26,037 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=0b0b0ae8de03,34589,1731875604293}]
2024-11-17T20:33:26,222 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-17T20:33:26,226 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49039, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-17T20:33:26,237 INFO  [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-17T20:33:26,238 INFO  [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-17T20:33:26,241 INFO  [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0b0b0ae8de03%2C34589%2C1731875604293.meta, suffix=.meta, logDir=hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/WALs/0b0b0ae8de03,34589,1731875604293, archiveDir=hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/oldWALs, maxLogs=32
2024-11-17T20:33:26,243 INFO  [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 0b0b0ae8de03%2C34589%2C1731875604293.meta.1731875606243.meta
2024-11-17T20:33:26,252 INFO  [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/WALs/0b0b0ae8de03,34589,1731875604293/0b0b0ae8de03%2C34589%2C1731875604293.meta.1731875606243.meta
2024-11-17T20:33:26,256 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33193:33193),(127.0.0.1/127.0.0.1:34187:34187)]
2024-11-17T20:33:26,260 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-17T20:33:26,262 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-17T20:33:26,264 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-17T20:33:26,268 INFO  [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-17T20:33:26,271 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-17T20:33:26,272 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T20:33:26,272 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-17T20:33:26,272 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-17T20:33:26,275 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-17T20:33:26,277 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-17T20:33:26,277 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:33:26,278 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T20:33:26,279 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-17T20:33:26,280 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-17T20:33:26,280 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:33:26,281 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T20:33:26,281 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-17T20:33:26,283 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-17T20:33:26,283 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:33:26,284 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T20:33:26,284 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-17T20:33:26,286 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-17T20:33:26,286 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:33:26,287 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T20:33:26,287 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-17T20:33:26,288 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/data/hbase/meta/1588230740
2024-11-17T20:33:26,291 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/data/hbase/meta/1588230740
2024-11-17T20:33:26,293 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-17T20:33:26,293 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-17T20:33:26,294 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-17T20:33:26,297 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-17T20:33:26,298 INFO  [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=843196, jitterRate=0.07217936217784882}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-17T20:33:26,299 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-17T20:33:26,300 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1731875606273Writing region info on filesystem at 1731875606273Initializing all the Stores at 1731875606275 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731875606275Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731875606275Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731875606275Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731875606275Cleaning up temporary data from old regions at 1731875606294 (+19 ms)Running coprocessor post-open hooks at 1731875606299 (+5 ms)Region opened successfully at 1731875606300 (+1 ms)
2024-11-17T20:33:26,305 INFO  [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731875606214
2024-11-17T20:33:26,316 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-17T20:33:26,316 INFO  [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-17T20:33:26,318 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=0b0b0ae8de03,34589,1731875604293
2024-11-17T20:33:26,321 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0b0b0ae8de03,34589,1731875604293, state=OPEN
2024-11-17T20:33:26,355 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34589-0x1014a807bd70001, quorum=127.0.0.1:51492, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-17T20:33:26,355 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35281-0x1014a807bd70000, quorum=127.0.0.1:51492, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-17T20:33:26,356 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-17T20:33:26,356 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-17T20:33:26,356 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=0b0b0ae8de03,34589,1731875604293
2024-11-17T20:33:26,361 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-17T20:33:26,361 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=0b0b0ae8de03,34589,1731875604293 in 320 msec
2024-11-17T20:33:26,368 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-17T20:33:26,368 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 830 msec
2024-11-17T20:33:26,369 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-17T20:33:26,369 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-17T20:33:26,385 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-17T20:33:26,386 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,34589,1731875604293, seqNum=-1]
2024-11-17T20:33:26,404 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-17T20:33:26,406 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53209, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-17T20:33:26,429 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0650 sec
2024-11-17T20:33:26,429 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731875606429, completionTime=-1
2024-11-17T20:33:26,432 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-17T20:33:26,432 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster...
2024-11-17T20:33:26,458 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1
2024-11-17T20:33:26,459 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731875666459
2024-11-17T20:33:26,459 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731875726459
2024-11-17T20:33:26,459 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 27 msec
2024-11-17T20:33:26,462 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0b0b0ae8de03,35281,1731875603390-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-17T20:33:26,462 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0b0b0ae8de03,35281,1731875603390-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T20:33:26,462 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0b0b0ae8de03,35281,1731875603390-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T20:33:26,464 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-0b0b0ae8de03:35281, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T20:33:26,464 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-17T20:33:26,465 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-17T20:33:26,471 DEBUG [master/0b0b0ae8de03:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-17T20:33:26,493 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.033sec
2024-11-17T20:33:26,495 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-17T20:33:26,496 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-17T20:33:26,497 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-17T20:33:26,498 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-17T20:33:26,498 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-17T20:33:26,498 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0b0b0ae8de03,35281,1731875603390-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-17T20:33:26,499 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0b0b0ae8de03,35281,1731875603390-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-17T20:33:26,507 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-17T20:33:26,508 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-17T20:33:26,508 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0b0b0ae8de03,35281,1731875603390-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T20:33:26,603 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e964458, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-17T20:33:26,606 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false
2024-11-17T20:33:26,606 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512
2024-11-17T20:33:26,633 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b0b0ae8de03,35281,-1 for getting cluster id
2024-11-17T20:33:26,636 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-17T20:33:26,644 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '900a8cee-cf87-4610-9c36-47ef82d3dd56'
2024-11-17T20:33:26,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-17T20:33:26,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "900a8cee-cf87-4610-9c36-47ef82d3dd56"

2024-11-17T20:33:26,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7076e47d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-17T20:33:26,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b0b0ae8de03,35281,-1]
2024-11-17T20:33:26,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-17T20:33:26,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T20:33:26,660 INFO  [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49684, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-17T20:33:26,663 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@500f1e50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-17T20:33:26,664 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-17T20:33:26,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,34589,1731875604293, seqNum=-1]
2024-11-17T20:33:26,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-17T20:33:26,674 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44286, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-17T20:33:26,692 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=0b0b0ae8de03,35281,1731875603390
2024-11-17T20:33:26,692 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T20:33:26,701 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-17T20:33:26,705 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-17T20:33:26,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 0b0b0ae8de03,35281,1731875603390
2024-11-17T20:33:26,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@260e034d
2024-11-17T20:33:26,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-17T20:33:26,715 INFO  [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49694, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-17T20:33:26,717 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35281 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-11-17T20:33:26,717 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35281 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-11-17T20:33:26,721 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35281 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testSlowSyncLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-17T20:33:26,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35281 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling
2024-11-17T20:33:26,732 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-11-17T20:33:26,733 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35281 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testSlowSyncLogRolling"
 procId is: 4
2024-11-17T20:33:26,734 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:33:26,736 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-17T20:33:26,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35281 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-17T20:33:26,788 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741835_1011 (size=389)
2024-11-17T20:33:26,788 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46521 is added to blk_1073741835_1011 (size=389)
2024-11-17T20:33:26,791 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8c4e53ab1a75e944dee323c61b2c115c, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1731875606717.8c4e53ab1a75e944dee323c61b2c115c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testSlowSyncLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d
2024-11-17T20:33:26,804 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46521 is added to blk_1073741836_1012 (size=72)
2024-11-17T20:33:26,804 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741836_1012 (size=72)
2024-11-17T20:33:26,805 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testSlowSyncLogRolling,,1731875606717.8c4e53ab1a75e944dee323c61b2c115c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T20:33:26,805 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1722): Closing 8c4e53ab1a75e944dee323c61b2c115c, disabling compactions & flushes
2024-11-17T20:33:26,805 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testSlowSyncLogRolling,,1731875606717.8c4e53ab1a75e944dee323c61b2c115c.
2024-11-17T20:33:26,805 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testSlowSyncLogRolling,,1731875606717.8c4e53ab1a75e944dee323c61b2c115c.
2024-11-17T20:33:26,805 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1731875606717.8c4e53ab1a75e944dee323c61b2c115c. after waiting 0 ms
2024-11-17T20:33:26,805 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1731875606717.8c4e53ab1a75e944dee323c61b2c115c.
2024-11-17T20:33:26,806 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testSlowSyncLogRolling,,1731875606717.8c4e53ab1a75e944dee323c61b2c115c.
2024-11-17T20:33:26,806 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8c4e53ab1a75e944dee323c61b2c115c:
Waiting for close lock at 1731875606805Disabling compacts and flushes for region at 1731875606805Disabling writes for close at 1731875606805Writing region close event to WAL at 1731875606805Closed at 1731875606805
2024-11-17T20:33:26,808 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ADD_TO_META
2024-11-17T20:33:26,811 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testSlowSyncLogRolling,,1731875606717.8c4e53ab1a75e944dee323c61b2c115c.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1731875606808"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731875606808"}]},"ts":"1731875606808"}
2024-11-17T20:33:26,817 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-17T20:33:26,819 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-17T20:33:26,822 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731875606819"}]},"ts":"1731875606819"}
2024-11-17T20:33:26,827 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLING in hbase:meta
2024-11-17T20:33:26,829 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=8c4e53ab1a75e944dee323c61b2c115c, ASSIGN}]
2024-11-17T20:33:26,832 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=8c4e53ab1a75e944dee323c61b2c115c, ASSIGN
2024-11-17T20:33:26,834 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=8c4e53ab1a75e944dee323c61b2c115c, ASSIGN; state=OFFLINE, location=0b0b0ae8de03,34589,1731875604293; forceNewPlan=false, retain=false
2024-11-17T20:33:26,987 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=8c4e53ab1a75e944dee323c61b2c115c, regionState=OPENING, regionLocation=0b0b0ae8de03,34589,1731875604293
2024-11-17T20:33:26,998 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=8c4e53ab1a75e944dee323c61b2c115c, ASSIGN because future has completed
2024-11-17T20:33:27,000 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8c4e53ab1a75e944dee323c61b2c115c, server=0b0b0ae8de03,34589,1731875604293}]
2024-11-17T20:33:27,167 INFO  [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testSlowSyncLogRolling,,1731875606717.8c4e53ab1a75e944dee323c61b2c115c.
2024-11-17T20:33:27,167 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 8c4e53ab1a75e944dee323c61b2c115c, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1731875606717.8c4e53ab1a75e944dee323c61b2c115c.', STARTKEY => '', ENDKEY => ''}
2024-11-17T20:33:27,168 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testSlowSyncLogRolling 8c4e53ab1a75e944dee323c61b2c115c
2024-11-17T20:33:27,168 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testSlowSyncLogRolling,,1731875606717.8c4e53ab1a75e944dee323c61b2c115c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T20:33:27,168 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 8c4e53ab1a75e944dee323c61b2c115c
2024-11-17T20:33:27,168 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 8c4e53ab1a75e944dee323c61b2c115c
2024-11-17T20:33:27,171 INFO  [StoreOpener-8c4e53ab1a75e944dee323c61b2c115c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 8c4e53ab1a75e944dee323c61b2c115c 
2024-11-17T20:33:27,173 INFO  [StoreOpener-8c4e53ab1a75e944dee323c61b2c115c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8c4e53ab1a75e944dee323c61b2c115c columnFamilyName info
2024-11-17T20:33:27,173 DEBUG [StoreOpener-8c4e53ab1a75e944dee323c61b2c115c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:33:27,174 INFO  [StoreOpener-8c4e53ab1a75e944dee323c61b2c115c-1 {}] regionserver.HStore(327): Store=8c4e53ab1a75e944dee323c61b2c115c/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-17T20:33:27,175 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 8c4e53ab1a75e944dee323c61b2c115c
2024-11-17T20:33:27,176 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/data/default/TestLogRolling-testSlowSyncLogRolling/8c4e53ab1a75e944dee323c61b2c115c
2024-11-17T20:33:27,177 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/data/default/TestLogRolling-testSlowSyncLogRolling/8c4e53ab1a75e944dee323c61b2c115c
2024-11-17T20:33:27,177 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 8c4e53ab1a75e944dee323c61b2c115c
2024-11-17T20:33:27,177 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 8c4e53ab1a75e944dee323c61b2c115c
2024-11-17T20:33:27,180 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 8c4e53ab1a75e944dee323c61b2c115c
2024-11-17T20:33:27,183 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/data/default/TestLogRolling-testSlowSyncLogRolling/8c4e53ab1a75e944dee323c61b2c115c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-17T20:33:27,184 INFO  [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 8c4e53ab1a75e944dee323c61b2c115c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=715562, jitterRate=-0.09011633694171906}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-17T20:33:27,184 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8c4e53ab1a75e944dee323c61b2c115c
2024-11-17T20:33:27,186 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 8c4e53ab1a75e944dee323c61b2c115c:
Running coprocessor pre-open hook at 1731875607168Writing region info on filesystem at 1731875607168Initializing all the Stores at 1731875607170 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731875607170Cleaning up temporary data from old regions at 1731875607177 (+7 ms)Running coprocessor post-open hooks at 1731875607184 (+7 ms)Region opened successfully at 1731875607186 (+2 ms)
2024-11-17T20:33:27,188 INFO  [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testSlowSyncLogRolling,,1731875606717.8c4e53ab1a75e944dee323c61b2c115c., pid=6, masterSystemTime=1731875607153
2024-11-17T20:33:27,191 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testSlowSyncLogRolling,,1731875606717.8c4e53ab1a75e944dee323c61b2c115c.
2024-11-17T20:33:27,191 INFO  [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testSlowSyncLogRolling,,1731875606717.8c4e53ab1a75e944dee323c61b2c115c.
2024-11-17T20:33:27,192 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=8c4e53ab1a75e944dee323c61b2c115c, regionState=OPEN, openSeqNum=2, regionLocation=0b0b0ae8de03,34589,1731875604293
2024-11-17T20:33:27,196 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8c4e53ab1a75e944dee323c61b2c115c, server=0b0b0ae8de03,34589,1731875604293 because future has completed
2024-11-17T20:33:27,202 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-17T20:33:27,202 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 8c4e53ab1a75e944dee323c61b2c115c, server=0b0b0ae8de03,34589,1731875604293 in 199 msec
2024-11-17T20:33:27,206 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-17T20:33:27,206 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=8c4e53ab1a75e944dee323c61b2c115c, ASSIGN in 373 msec
2024-11-17T20:33:27,208 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-17T20:33:27,208 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731875607208"}]},"ts":"1731875607208"}
2024-11-17T20:33:27,211 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLED in hbase:meta
2024-11-17T20:33:27,213 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_POST_OPERATION
2024-11-17T20:33:27,217 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling in 491 msec
2024-11-17T20:33:31,790 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties
2024-11-17T20:33:31,838 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-17T20:33:31,840 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testSlowSyncLogRolling'
2024-11-17T20:33:33,804 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-17T20:33:33,805 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-11-17T20:33:33,810 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling
2024-11-17T20:33:33,810 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling Metrics about Tables on a single HBase RegionServer
2024-11-17T20:33:33,814 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-17T20:33:33,814 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-11-17T20:33:33,815 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-11-17T20:33:33,815 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer
2024-11-17T20:33:36,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35281 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-17T20:33:36,848 INFO  [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testSlowSyncLogRolling completed
2024-11-17T20:33:36,851 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testSlowSyncLogRolling,, stopping at row=TestLogRolling-testSlowSyncLogRolling ,, for max=2147483647 with caching=100
2024-11-17T20:33:36,857 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testSlowSyncLogRolling
2024-11-17T20:33:36,858 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testSlowSyncLogRolling,,1731875606717.8c4e53ab1a75e944dee323c61b2c115c.
2024-11-17T20:33:36,859 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0b0b0ae8de03%2C34589%2C1731875604293.1731875616859
2024-11-17T20:33:36,869 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:33:36,869 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:33:36,869 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:33:36,869 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:33:36,869 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:33:36,870 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/WALs/0b0b0ae8de03,34589,1731875604293/0b0b0ae8de03%2C34589%2C1731875604293.1731875605763 with entries=1, filesize=443 B; new WAL /user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/WALs/0b0b0ae8de03,34589,1731875604293/0b0b0ae8de03%2C34589%2C1731875604293.1731875616859
2024-11-17T20:33:36,873 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741833_1009 (size=451)
2024-11-17T20:33:36,873 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46521 is added to blk_1073741833_1009 (size=451)
2024-11-17T20:33:36,879 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/WALs/0b0b0ae8de03,34589,1731875604293/0b0b0ae8de03%2C34589%2C1731875604293.1731875605763 to hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/oldWALs/0b0b0ae8de03%2C34589%2C1731875604293.1731875605763
2024-11-17T20:33:36,879 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33193:33193),(127.0.0.1/127.0.0.1:34187:34187)]
2024-11-17T20:33:36,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testSlowSyncLogRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testSlowSyncLogRolling,,1731875606717.8c4e53ab1a75e944dee323c61b2c115c., hostname=0b0b0ae8de03,34589,1731875604293, seqNum=2]
2024-11-17T20:33:48,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34589 {}] regionserver.HRegion(8855): Flush requested on 8c4e53ab1a75e944dee323c61b2c115c
2024-11-17T20:33:48,928 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8c4e53ab1a75e944dee323c61b2c115c 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-17T20:33:48,997 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/data/default/TestLogRolling-testSlowSyncLogRolling/8c4e53ab1a75e944dee323c61b2c115c/.tmp/info/37b64bf44cf0427387fde3869a974442 is 1080, key is row0001/info:/1731875616890/Put/seqid=0
2024-11-17T20:33:49,010 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741838_1014 (size=12509)
2024-11-17T20:33:49,010 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46521 is added to blk_1073741838_1014 (size=12509)
2024-11-17T20:33:49,011 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/data/default/TestLogRolling-testSlowSyncLogRolling/8c4e53ab1a75e944dee323c61b2c115c/.tmp/info/37b64bf44cf0427387fde3869a974442
2024-11-17T20:33:49,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/data/default/TestLogRolling-testSlowSyncLogRolling/8c4e53ab1a75e944dee323c61b2c115c/.tmp/info/37b64bf44cf0427387fde3869a974442 as hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/data/default/TestLogRolling-testSlowSyncLogRolling/8c4e53ab1a75e944dee323c61b2c115c/info/37b64bf44cf0427387fde3869a974442
2024-11-17T20:33:49,075 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/data/default/TestLogRolling-testSlowSyncLogRolling/8c4e53ab1a75e944dee323c61b2c115c/info/37b64bf44cf0427387fde3869a974442, entries=7, sequenceid=11, filesize=12.2 K
2024-11-17T20:33:49,083 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 8c4e53ab1a75e944dee323c61b2c115c in 154ms, sequenceid=11, compaction requested=false
2024-11-17T20:33:49,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8c4e53ab1a75e944dee323c61b2c115c:

2024-11-17T20:33:52,429 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-17T20:33:56,938 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0b0b0ae8de03%2C34589%2C1731875604293.1731875636938
2024-11-17T20:33:57,146 INFO  [Time-limited test {}] wal.AbstractFSWAL(1368): Slow sync cost: 205 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34923,DS-53159149-da7e-4ad8-9376-eaab27d68ac7,DISK], DatanodeInfoWithStorage[127.0.0.1:46521,DS-5f2c1e58-7b56-4468-be16-067721c1967c,DISK]]
2024-11-17T20:33:57,147 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:33:57,147 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:33:57,147 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:33:57,147 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:33:57,147 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:33:57,147 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/WALs/0b0b0ae8de03,34589,1731875604293/0b0b0ae8de03%2C34589%2C1731875604293.1731875616859 with entries=12, filesize=12.10 KB; new WAL /user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/WALs/0b0b0ae8de03,34589,1731875604293/0b0b0ae8de03%2C34589%2C1731875604293.1731875636938
2024-11-17T20:33:57,150 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46521 is added to blk_1073741837_1013 (size=12399)
2024-11-17T20:33:57,151 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741837_1013 (size=12399)
2024-11-17T20:33:57,160 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34187:34187),(127.0.0.1/127.0.0.1:33193:33193)]
2024-11-17T20:33:57,365 INFO  [FSHLog-0-hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d-prefix:0b0b0ae8de03,34589,1731875604293 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46521,DS-5f2c1e58-7b56-4468-be16-067721c1967c,DISK], DatanodeInfoWithStorage[127.0.0.1:34923,DS-53159149-da7e-4ad8-9376-eaab27d68ac7,DISK]]
2024-11-17T20:33:59,603 INFO  [FSHLog-0-hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d-prefix:0b0b0ae8de03,34589,1731875604293 {}] wal.AbstractFSWAL(1368): Slow sync cost: 235 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46521,DS-5f2c1e58-7b56-4468-be16-067721c1967c,DISK], DatanodeInfoWithStorage[127.0.0.1:34923,DS-53159149-da7e-4ad8-9376-eaab27d68ac7,DISK]]
2024-11-17T20:34:01,810 INFO  [FSHLog-0-hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d-prefix:0b0b0ae8de03,34589,1731875604293 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46521,DS-5f2c1e58-7b56-4468-be16-067721c1967c,DISK], DatanodeInfoWithStorage[127.0.0.1:34923,DS-53159149-da7e-4ad8-9376-eaab27d68ac7,DISK]]
2024-11-17T20:34:04,016 INFO  [FSHLog-0-hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d-prefix:0b0b0ae8de03,34589,1731875604293 {}] wal.AbstractFSWAL(1368): Slow sync cost: 203 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46521,DS-5f2c1e58-7b56-4468-be16-067721c1967c,DISK], DatanodeInfoWithStorage[127.0.0.1:34923,DS-53159149-da7e-4ad8-9376-eaab27d68ac7,DISK]]
2024-11-17T20:34:04,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34589 {}] regionserver.HRegion(8855): Flush requested on 8c4e53ab1a75e944dee323c61b2c115c
2024-11-17T20:34:04,017 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8c4e53ab1a75e944dee323c61b2c115c 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-17T20:34:04,222 INFO  [FSHLog-0-hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d-prefix:0b0b0ae8de03,34589,1731875604293 {}] wal.AbstractFSWAL(1368): Slow sync cost: 202 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46521,DS-5f2c1e58-7b56-4468-be16-067721c1967c,DISK], DatanodeInfoWithStorage[127.0.0.1:34923,DS-53159149-da7e-4ad8-9376-eaab27d68ac7,DISK]]
2024-11-17T20:34:04,234 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/data/default/TestLogRolling-testSlowSyncLogRolling/8c4e53ab1a75e944dee323c61b2c115c/.tmp/info/8bfc6398497b44f79d4a884990d02724 is 1080, key is row0008/info:/1731875630927/Put/seqid=0
2024-11-17T20:34:04,242 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46521 is added to blk_1073741840_1016 (size=12509)
2024-11-17T20:34:04,243 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741840_1016 (size=12509)
2024-11-17T20:34:04,244 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/data/default/TestLogRolling-testSlowSyncLogRolling/8c4e53ab1a75e944dee323c61b2c115c/.tmp/info/8bfc6398497b44f79d4a884990d02724
2024-11-17T20:34:04,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/data/default/TestLogRolling-testSlowSyncLogRolling/8c4e53ab1a75e944dee323c61b2c115c/.tmp/info/8bfc6398497b44f79d4a884990d02724 as hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/data/default/TestLogRolling-testSlowSyncLogRolling/8c4e53ab1a75e944dee323c61b2c115c/info/8bfc6398497b44f79d4a884990d02724
2024-11-17T20:34:04,263 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/data/default/TestLogRolling-testSlowSyncLogRolling/8c4e53ab1a75e944dee323c61b2c115c/info/8bfc6398497b44f79d4a884990d02724, entries=7, sequenceid=21, filesize=12.2 K
2024-11-17T20:34:04,466 INFO  [FSHLog-0-hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d-prefix:0b0b0ae8de03,34589,1731875604293 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46521,DS-5f2c1e58-7b56-4468-be16-067721c1967c,DISK], DatanodeInfoWithStorage[127.0.0.1:34923,DS-53159149-da7e-4ad8-9376-eaab27d68ac7,DISK]]
2024-11-17T20:34:04,466 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 8c4e53ab1a75e944dee323c61b2c115c in 450ms, sequenceid=21, compaction requested=false
2024-11-17T20:34:04,466 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8c4e53ab1a75e944dee323c61b2c115c:

2024-11-17T20:34:04,467 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=24.4 K, sizeToCheck=16.0 K
2024-11-17T20:34:04,467 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-17T20:34:04,468 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/data/default/TestLogRolling-testSlowSyncLogRolling/8c4e53ab1a75e944dee323c61b2c115c/info/37b64bf44cf0427387fde3869a974442 because midkey is the same as first or last row
2024-11-17T20:34:06,223 INFO  [FSHLog-0-hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d-prefix:0b0b0ae8de03,34589,1731875604293 {}] wal.AbstractFSWAL(1368): Slow sync cost: 202 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46521,DS-5f2c1e58-7b56-4468-be16-067721c1967c,DISK], DatanodeInfoWithStorage[127.0.0.1:34923,DS-53159149-da7e-4ad8-9376-eaab27d68ac7,DISK]]
2024-11-17T20:34:06,550 INFO  [master/0b0b0ae8de03:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-11-17T20:34:06,550 INFO  [master/0b0b0ae8de03:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-11-17T20:34:08,428 INFO  [FSHLog-0-hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d-prefix:0b0b0ae8de03,34589,1731875604293 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46521,DS-5f2c1e58-7b56-4468-be16-067721c1967c,DISK], DatanodeInfoWithStorage[127.0.0.1:34923,DS-53159149-da7e-4ad8-9376-eaab27d68ac7,DISK]]
2024-11-17T20:34:08,430 WARN  [FSHLog-0-hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d-prefix:0b0b0ae8de03,34589,1731875604293 {}] wal.AbstractFSWAL(2201): Requesting log roll because we exceeded slow sync threshold; count=8, threshold=5, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46521,DS-5f2c1e58-7b56-4468-be16-067721c1967c,DISK], DatanodeInfoWithStorage[127.0.0.1:34923,DS-53159149-da7e-4ad8-9376-eaab27d68ac7,DISK]]
2024-11-17T20:34:08,431 DEBUG [regionserver/0b0b0ae8de03:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 0b0b0ae8de03%2C34589%2C1731875604293:(num 1731875636938) roll requested
2024-11-17T20:34:08,432 INFO  [regionserver/0b0b0ae8de03:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0b0b0ae8de03%2C34589%2C1731875604293.1731875648432
2024-11-17T20:34:08,642 INFO  [regionserver/0b0b0ae8de03:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 208 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46521,DS-5f2c1e58-7b56-4468-be16-067721c1967c,DISK], DatanodeInfoWithStorage[127.0.0.1:34923,DS-53159149-da7e-4ad8-9376-eaab27d68ac7,DISK]]
2024-11-17T20:34:08,642 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:34:08,643 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:34:08,643 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:34:08,643 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:34:08,643 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:34:08,644 INFO  [regionserver/0b0b0ae8de03:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/WALs/0b0b0ae8de03,34589,1731875604293/0b0b0ae8de03%2C34589%2C1731875604293.1731875636938 with entries=8, filesize=7.55 KB; new WAL /user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/WALs/0b0b0ae8de03,34589,1731875604293/0b0b0ae8de03%2C34589%2C1731875604293.1731875648432
2024-11-17T20:34:08,645 DEBUG [regionserver/0b0b0ae8de03:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33193:33193),(127.0.0.1/127.0.0.1:34187:34187)]
2024-11-17T20:34:08,645 DEBUG [regionserver/0b0b0ae8de03:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/WALs/0b0b0ae8de03,34589,1731875604293/0b0b0ae8de03%2C34589%2C1731875604293.1731875636938 is not closed yet, will try archiving it next time
2024-11-17T20:34:08,645 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/WALs/0b0b0ae8de03,34589,1731875604293/0b0b0ae8de03%2C34589%2C1731875604293.1731875616859 to hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/oldWALs/0b0b0ae8de03%2C34589%2C1731875604293.1731875616859
2024-11-17T20:34:08,647 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741839_1015 (size=7739)
2024-11-17T20:34:08,648 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46521 is added to blk_1073741839_1015 (size=7739)
2024-11-17T20:34:10,635 INFO  [FSHLog-0-hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d-prefix:0b0b0ae8de03,34589,1731875604293 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34923,DS-53159149-da7e-4ad8-9376-eaab27d68ac7,DISK], DatanodeInfoWithStorage[127.0.0.1:46521,DS-5f2c1e58-7b56-4468-be16-067721c1967c,DISK]]
2024-11-17T20:34:12,168 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 8c4e53ab1a75e944dee323c61b2c115c, had cached 0 bytes from a total of 25018
2024-11-17T20:34:12,841 INFO  [FSHLog-0-hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d-prefix:0b0b0ae8de03,34589,1731875604293 {}] wal.AbstractFSWAL(1368): Slow sync cost: 202 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34923,DS-53159149-da7e-4ad8-9376-eaab27d68ac7,DISK], DatanodeInfoWithStorage[127.0.0.1:46521,DS-5f2c1e58-7b56-4468-be16-067721c1967c,DISK]]
2024-11-17T20:34:15,046 INFO  [FSHLog-0-hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d-prefix:0b0b0ae8de03,34589,1731875604293 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34923,DS-53159149-da7e-4ad8-9376-eaab27d68ac7,DISK], DatanodeInfoWithStorage[127.0.0.1:46521,DS-5f2c1e58-7b56-4468-be16-067721c1967c,DISK]]
2024-11-17T20:34:17,251 INFO  [FSHLog-0-hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d-prefix:0b0b0ae8de03,34589,1731875604293 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34923,DS-53159149-da7e-4ad8-9376-eaab27d68ac7,DISK], DatanodeInfoWithStorage[127.0.0.1:46521,DS-5f2c1e58-7b56-4468-be16-067721c1967c,DISK]]
2024-11-17T20:34:19,253 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2024-11-17T20:34:19,254 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0b0b0ae8de03%2C34589%2C1731875604293.1731875659253
2024-11-17T20:34:22,429 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-17T20:34:24,266 INFO  [Time-limited test {}] wal.AbstractFSWAL(1368): Slow sync cost: 5009 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34923,DS-53159149-da7e-4ad8-9376-eaab27d68ac7,DISK], DatanodeInfoWithStorage[127.0.0.1:46521,DS-5f2c1e58-7b56-4468-be16-067721c1967c,DISK]]
2024-11-17T20:34:24,269 WARN  [Time-limited test {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5009 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34923,DS-53159149-da7e-4ad8-9376-eaab27d68ac7,DISK], DatanodeInfoWithStorage[127.0.0.1:46521,DS-5f2c1e58-7b56-4468-be16-067721c1967c,DISK]]
2024-11-17T20:34:24,269 DEBUG [regionserver/0b0b0ae8de03:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 0b0b0ae8de03%2C34589%2C1731875604293:(num 1731875659253) roll requested
2024-11-17T20:34:24,269 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:34:24,269 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:34:24,270 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:34:24,270 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:34:24,270 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:34:24,270 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/WALs/0b0b0ae8de03,34589,1731875604293/0b0b0ae8de03%2C34589%2C1731875604293.1731875648432 with entries=4, filesize=4.63 KB; new WAL /user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/WALs/0b0b0ae8de03,34589,1731875604293/0b0b0ae8de03%2C34589%2C1731875604293.1731875659253
2024-11-17T20:34:24,271 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33193:33193),(127.0.0.1/127.0.0.1:34187:34187)]
2024-11-17T20:34:24,271 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/WALs/0b0b0ae8de03,34589,1731875604293/0b0b0ae8de03%2C34589%2C1731875604293.1731875648432 is not closed yet, will try archiving it next time
2024-11-17T20:34:24,272 INFO  [regionserver/0b0b0ae8de03:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0b0b0ae8de03%2C34589%2C1731875604293.1731875664271
2024-11-17T20:34:24,273 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46521 is added to blk_1073741841_1017 (size=4753)
2024-11-17T20:34:24,275 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741841_1017 (size=4753)
2024-11-17T20:34:29,276 INFO  [FSHLog-0-hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d-prefix:0b0b0ae8de03,34589,1731875604293 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5002 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34923,DS-53159149-da7e-4ad8-9376-eaab27d68ac7,DISK], DatanodeInfoWithStorage[127.0.0.1:46521,DS-5f2c1e58-7b56-4468-be16-067721c1967c,DISK]]
2024-11-17T20:34:29,277 WARN  [FSHLog-0-hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d-prefix:0b0b0ae8de03,34589,1731875604293 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5002 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34923,DS-53159149-da7e-4ad8-9376-eaab27d68ac7,DISK], DatanodeInfoWithStorage[127.0.0.1:46521,DS-5f2c1e58-7b56-4468-be16-067721c1967c,DISK]]
2024-11-17T20:34:29,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34589 {}] regionserver.HRegion(8855): Flush requested on 8c4e53ab1a75e944dee323c61b2c115c
2024-11-17T20:34:29,277 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8c4e53ab1a75e944dee323c61b2c115c 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-17T20:34:29,286 INFO  [regionserver/0b0b0ae8de03:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 5010 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34923,DS-53159149-da7e-4ad8-9376-eaab27d68ac7,DISK], DatanodeInfoWithStorage[127.0.0.1:46521,DS-5f2c1e58-7b56-4468-be16-067721c1967c,DISK]]
2024-11-17T20:34:29,286 WARN  [regionserver/0b0b0ae8de03:0.logRoller {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5010 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34923,DS-53159149-da7e-4ad8-9376-eaab27d68ac7,DISK], DatanodeInfoWithStorage[127.0.0.1:46521,DS-5f2c1e58-7b56-4468-be16-067721c1967c,DISK]]
2024-11-17T20:34:31,278 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2024-11-17T20:34:34,281 INFO  [FSHLog-0-hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d-prefix:0b0b0ae8de03,34589,1731875604293 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5001 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34923,DS-53159149-da7e-4ad8-9376-eaab27d68ac7,DISK], DatanodeInfoWithStorage[127.0.0.1:46521,DS-5f2c1e58-7b56-4468-be16-067721c1967c,DISK]]
2024-11-17T20:34:34,281 WARN  [FSHLog-0-hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d-prefix:0b0b0ae8de03,34589,1731875604293 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5001 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34923,DS-53159149-da7e-4ad8-9376-eaab27d68ac7,DISK], DatanodeInfoWithStorage[127.0.0.1:46521,DS-5f2c1e58-7b56-4468-be16-067721c1967c,DISK]]
2024-11-17T20:34:34,282 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:34:34,282 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:34:34,282 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:34:34,283 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:34:34,283 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:34:34,283 INFO  [regionserver/0b0b0ae8de03:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/WALs/0b0b0ae8de03,34589,1731875604293/0b0b0ae8de03%2C34589%2C1731875604293.1731875659253 with entries=2, filesize=1.52 KB; new WAL /user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/WALs/0b0b0ae8de03,34589,1731875604293/0b0b0ae8de03%2C34589%2C1731875604293.1731875664271
2024-11-17T20:34:34,285 DEBUG [regionserver/0b0b0ae8de03:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33193:33193),(127.0.0.1/127.0.0.1:34187:34187)]
2024-11-17T20:34:34,286 DEBUG [regionserver/0b0b0ae8de03:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/WALs/0b0b0ae8de03,34589,1731875604293/0b0b0ae8de03%2C34589%2C1731875604293.1731875659253 is not closed yet, will try archiving it next time
2024-11-17T20:34:34,286 DEBUG [regionserver/0b0b0ae8de03:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 0b0b0ae8de03%2C34589%2C1731875604293:(num 1731875664271) roll requested
2024-11-17T20:34:34,287 INFO  [regionserver/0b0b0ae8de03:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0b0b0ae8de03%2C34589%2C1731875604293.1731875674286
2024-11-17T20:34:34,287 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46521 is added to blk_1073741842_1018 (size=1569)
2024-11-17T20:34:34,288 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741842_1018 (size=1569)
2024-11-17T20:34:34,292 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/data/default/TestLogRolling-testSlowSyncLogRolling/8c4e53ab1a75e944dee323c61b2c115c/.tmp/info/f61d5fdc21d044f2a34befa80baaf612 is 1080, key is row0015/info:/1731875646020/Put/seqid=0
2024-11-17T20:34:34,297 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741844_1020 (size=12509)
2024-11-17T20:34:34,298 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46521 is added to blk_1073741844_1020 (size=12509)
2024-11-17T20:34:34,298 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=31 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/data/default/TestLogRolling-testSlowSyncLogRolling/8c4e53ab1a75e944dee323c61b2c115c/.tmp/info/f61d5fdc21d044f2a34befa80baaf612
2024-11-17T20:34:34,308 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/data/default/TestLogRolling-testSlowSyncLogRolling/8c4e53ab1a75e944dee323c61b2c115c/.tmp/info/f61d5fdc21d044f2a34befa80baaf612 as hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/data/default/TestLogRolling-testSlowSyncLogRolling/8c4e53ab1a75e944dee323c61b2c115c/info/f61d5fdc21d044f2a34befa80baaf612
2024-11-17T20:34:34,318 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/data/default/TestLogRolling-testSlowSyncLogRolling/8c4e53ab1a75e944dee323c61b2c115c/info/f61d5fdc21d044f2a34befa80baaf612, entries=7, sequenceid=31, filesize=12.2 K
2024-11-17T20:34:39,321 INFO  [FSHLog-0-hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d-prefix:0b0b0ae8de03,34589,1731875604293 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5002 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34923,DS-53159149-da7e-4ad8-9376-eaab27d68ac7,DISK], DatanodeInfoWithStorage[127.0.0.1:46521,DS-5f2c1e58-7b56-4468-be16-067721c1967c,DISK]]
2024-11-17T20:34:39,321 WARN  [FSHLog-0-hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d-prefix:0b0b0ae8de03,34589,1731875604293 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5002 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34923,DS-53159149-da7e-4ad8-9376-eaab27d68ac7,DISK], DatanodeInfoWithStorage[127.0.0.1:46521,DS-5f2c1e58-7b56-4468-be16-067721c1967c,DISK]]
2024-11-17T20:34:39,321 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 8c4e53ab1a75e944dee323c61b2c115c in 10044ms, sequenceid=31, compaction requested=true
2024-11-17T20:34:39,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8c4e53ab1a75e944dee323c61b2c115c:

2024-11-17T20:34:39,322 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=36.6 K, sizeToCheck=16.0 K
2024-11-17T20:34:39,322 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-17T20:34:39,323 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/data/default/TestLogRolling-testSlowSyncLogRolling/8c4e53ab1a75e944dee323c61b2c115c/info/37b64bf44cf0427387fde3869a974442 because midkey is the same as first or last row
2024-11-17T20:34:39,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8c4e53ab1a75e944dee323c61b2c115c:info, priority=-2147483648, current under compaction store size is 1
2024-11-17T20:34:39,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-17T20:34:39,328 DEBUG [RS:0;0b0b0ae8de03:34589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-17T20:34:39,332 DEBUG [RS:0;0b0b0ae8de03:34589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 37527 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-17T20:34:39,333 DEBUG [RS:0;0b0b0ae8de03:34589-shortCompactions-0 {}] regionserver.HStore(1541): 8c4e53ab1a75e944dee323c61b2c115c/info is initiating minor compaction (all files)
2024-11-17T20:34:39,333 INFO  [RS:0;0b0b0ae8de03:34589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8c4e53ab1a75e944dee323c61b2c115c/info in TestLogRolling-testSlowSyncLogRolling,,1731875606717.8c4e53ab1a75e944dee323c61b2c115c.
2024-11-17T20:34:39,334 INFO  [RS:0;0b0b0ae8de03:34589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/data/default/TestLogRolling-testSlowSyncLogRolling/8c4e53ab1a75e944dee323c61b2c115c/info/37b64bf44cf0427387fde3869a974442, hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/data/default/TestLogRolling-testSlowSyncLogRolling/8c4e53ab1a75e944dee323c61b2c115c/info/8bfc6398497b44f79d4a884990d02724, hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/data/default/TestLogRolling-testSlowSyncLogRolling/8c4e53ab1a75e944dee323c61b2c115c/info/f61d5fdc21d044f2a34befa80baaf612] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/data/default/TestLogRolling-testSlowSyncLogRolling/8c4e53ab1a75e944dee323c61b2c115c/.tmp, totalSize=36.6 K
2024-11-17T20:34:39,335 DEBUG [RS:0;0b0b0ae8de03:34589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37b64bf44cf0427387fde3869a974442, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731875616890
2024-11-17T20:34:39,336 DEBUG [RS:0;0b0b0ae8de03:34589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8bfc6398497b44f79d4a884990d02724, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=21, earliestPutTs=1731875630927
2024-11-17T20:34:39,336 DEBUG [RS:0;0b0b0ae8de03:34589-shortCompactions-0 {}] compactions.Compactor(225): Compacting f61d5fdc21d044f2a34befa80baaf612, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=31, earliestPutTs=1731875646020
2024-11-17T20:34:39,354 INFO  [regionserver/0b0b0ae8de03:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 5063 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34923,DS-53159149-da7e-4ad8-9376-eaab27d68ac7,DISK], DatanodeInfoWithStorage[127.0.0.1:46521,DS-5f2c1e58-7b56-4468-be16-067721c1967c,DISK]]
2024-11-17T20:34:39,354 WARN  [regionserver/0b0b0ae8de03:0.logRoller {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5063 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34923,DS-53159149-da7e-4ad8-9376-eaab27d68ac7,DISK], DatanodeInfoWithStorage[127.0.0.1:46521,DS-5f2c1e58-7b56-4468-be16-067721c1967c,DISK]]
2024-11-17T20:34:39,354 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:34:39,354 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:34:39,355 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:34:39,355 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:34:39,355 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:34:39,355 INFO  [regionserver/0b0b0ae8de03:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/WALs/0b0b0ae8de03,34589,1731875604293/0b0b0ae8de03%2C34589%2C1731875604293.1731875664271 with entries=1, filesize=430 B; new WAL /user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/WALs/0b0b0ae8de03,34589,1731875604293/0b0b0ae8de03%2C34589%2C1731875604293.1731875674286
2024-11-17T20:34:39,357 DEBUG [regionserver/0b0b0ae8de03:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33193:33193),(127.0.0.1/127.0.0.1:34187:34187)]
2024-11-17T20:34:39,357 DEBUG [regionserver/0b0b0ae8de03:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/WALs/0b0b0ae8de03,34589,1731875604293/0b0b0ae8de03%2C34589%2C1731875604293.1731875664271 is not closed yet, will try archiving it next time
2024-11-17T20:34:39,357 DEBUG [regionserver/0b0b0ae8de03:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 0b0b0ae8de03%2C34589%2C1731875604293:(num 1731875674286) roll requested
2024-11-17T20:34:39,357 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/WALs/0b0b0ae8de03,34589,1731875604293/0b0b0ae8de03%2C34589%2C1731875604293.1731875636938 to hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/oldWALs/0b0b0ae8de03%2C34589%2C1731875604293.1731875636938
2024-11-17T20:34:39,357 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0b0b0ae8de03%2C34589%2C1731875604293.1731875679357
2024-11-17T20:34:39,358 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46521 is added to blk_1073741843_1019 (size=438)
2024-11-17T20:34:39,359 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741843_1019 (size=438)
2024-11-17T20:34:39,361 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/WALs/0b0b0ae8de03,34589,1731875604293/0b0b0ae8de03%2C34589%2C1731875604293.1731875648432 to hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/oldWALs/0b0b0ae8de03%2C34589%2C1731875604293.1731875648432
2024-11-17T20:34:39,363 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/WALs/0b0b0ae8de03,34589,1731875604293/0b0b0ae8de03%2C34589%2C1731875604293.1731875659253 to hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/oldWALs/0b0b0ae8de03%2C34589%2C1731875604293.1731875659253
2024-11-17T20:34:39,364 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/WALs/0b0b0ae8de03,34589,1731875604293/0b0b0ae8de03%2C34589%2C1731875604293.1731875664271 to hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/oldWALs/0b0b0ae8de03%2C34589%2C1731875604293.1731875664271
2024-11-17T20:34:39,373 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:34:39,373 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:34:39,373 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:34:39,373 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:34:39,373 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:34:39,374 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/WALs/0b0b0ae8de03,34589,1731875604293/0b0b0ae8de03%2C34589%2C1731875604293.1731875674286 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/WALs/0b0b0ae8de03,34589,1731875604293/0b0b0ae8de03%2C34589%2C1731875604293.1731875679357
2024-11-17T20:34:39,376 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46521 is added to blk_1073741845_1021 (size=93)
2024-11-17T20:34:39,377 INFO  [RS:0;0b0b0ae8de03:34589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8c4e53ab1a75e944dee323c61b2c115c#info#compaction#3 average throughput is 10.77 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-17T20:34:39,377 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741845_1021 (size=93)
2024-11-17T20:34:39,378 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/WALs/0b0b0ae8de03,34589,1731875604293/0b0b0ae8de03%2C34589%2C1731875604293.1731875674286 to hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/oldWALs/0b0b0ae8de03%2C34589%2C1731875604293.1731875674286
2024-11-17T20:34:39,378 DEBUG [RS:0;0b0b0ae8de03:34589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/data/default/TestLogRolling-testSlowSyncLogRolling/8c4e53ab1a75e944dee323c61b2c115c/.tmp/info/b090166d4e784ac58c7abfcd476a394f is 1080, key is row0001/info:/1731875616890/Put/seqid=0
2024-11-17T20:34:39,384 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34187:34187),(127.0.0.1/127.0.0.1:33193:33193)]
2024-11-17T20:34:39,385 INFO  [regionserver/0b0b0ae8de03:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0b0b0ae8de03%2C34589%2C1731875604293.1731875679384
2024-11-17T20:34:39,388 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741847_1023 (size=27710)
2024-11-17T20:34:39,388 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46521 is added to blk_1073741847_1023 (size=27710)
2024-11-17T20:34:39,396 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:34:39,396 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:34:39,396 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:34:39,396 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:34:39,396 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:34:39,396 INFO  [regionserver/0b0b0ae8de03:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/WALs/0b0b0ae8de03,34589,1731875604293/0b0b0ae8de03%2C34589%2C1731875604293.1731875679357 with entries=1, filesize=1.22 KB; new WAL /user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/WALs/0b0b0ae8de03,34589,1731875604293/0b0b0ae8de03%2C34589%2C1731875604293.1731875679384
2024-11-17T20:34:39,398 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741846_1022 (size=1258)
2024-11-17T20:34:39,398 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46521 is added to blk_1073741846_1022 (size=1258)
2024-11-17T20:34:39,400 DEBUG [RS:0;0b0b0ae8de03:34589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/data/default/TestLogRolling-testSlowSyncLogRolling/8c4e53ab1a75e944dee323c61b2c115c/.tmp/info/b090166d4e784ac58c7abfcd476a394f as hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/data/default/TestLogRolling-testSlowSyncLogRolling/8c4e53ab1a75e944dee323c61b2c115c/info/b090166d4e784ac58c7abfcd476a394f
2024-11-17T20:34:39,401 DEBUG [regionserver/0b0b0ae8de03:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33193:33193),(127.0.0.1/127.0.0.1:34187:34187)]
2024-11-17T20:34:39,415 INFO  [RS:0;0b0b0ae8de03:34589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8c4e53ab1a75e944dee323c61b2c115c/info of 8c4e53ab1a75e944dee323c61b2c115c into b090166d4e784ac58c7abfcd476a394f(size=27.1 K), total size for store is 27.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-17T20:34:39,415 DEBUG [RS:0;0b0b0ae8de03:34589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8c4e53ab1a75e944dee323c61b2c115c:

2024-11-17T20:34:39,417 INFO  [RS:0;0b0b0ae8de03:34589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testSlowSyncLogRolling,,1731875606717.8c4e53ab1a75e944dee323c61b2c115c., storeName=8c4e53ab1a75e944dee323c61b2c115c/info, priority=13, startTime=1731875679326; duration=0sec
2024-11-17T20:34:39,417 DEBUG [RS:0;0b0b0ae8de03:34589-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K
2024-11-17T20:34:39,417 DEBUG [RS:0;0b0b0ae8de03:34589-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-17T20:34:39,417 DEBUG [RS:0;0b0b0ae8de03:34589-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/data/default/TestLogRolling-testSlowSyncLogRolling/8c4e53ab1a75e944dee323c61b2c115c/info/b090166d4e784ac58c7abfcd476a394f because midkey is the same as first or last row
2024-11-17T20:34:39,417 DEBUG [RS:0;0b0b0ae8de03:34589-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K
2024-11-17T20:34:39,417 DEBUG [RS:0;0b0b0ae8de03:34589-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-17T20:34:39,417 DEBUG [RS:0;0b0b0ae8de03:34589-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/data/default/TestLogRolling-testSlowSyncLogRolling/8c4e53ab1a75e944dee323c61b2c115c/info/b090166d4e784ac58c7abfcd476a394f because midkey is the same as first or last row
2024-11-17T20:34:39,418 DEBUG [RS:0;0b0b0ae8de03:34589-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K
2024-11-17T20:34:39,418 DEBUG [RS:0;0b0b0ae8de03:34589-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-17T20:34:39,418 DEBUG [RS:0;0b0b0ae8de03:34589-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/data/default/TestLogRolling-testSlowSyncLogRolling/8c4e53ab1a75e944dee323c61b2c115c/info/b090166d4e784ac58c7abfcd476a394f because midkey is the same as first or last row
2024-11-17T20:34:39,418 DEBUG [RS:0;0b0b0ae8de03:34589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-17T20:34:39,418 DEBUG [RS:0;0b0b0ae8de03:34589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8c4e53ab1a75e944dee323c61b2c115c:info
2024-11-17T20:34:51,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34589 {}] regionserver.HRegion(8855): Flush requested on 8c4e53ab1a75e944dee323c61b2c115c
2024-11-17T20:34:51,417 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8c4e53ab1a75e944dee323c61b2c115c 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-17T20:34:51,425 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/data/default/TestLogRolling-testSlowSyncLogRolling/8c4e53ab1a75e944dee323c61b2c115c/.tmp/info/133a107c0f7a43b48852c5248e3955a9 is 1080, key is row0022/info:/1731875679386/Put/seqid=0
2024-11-17T20:34:51,432 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741849_1025 (size=12509)
2024-11-17T20:34:51,432 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46521 is added to blk_1073741849_1025 (size=12509)
2024-11-17T20:34:51,433 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/data/default/TestLogRolling-testSlowSyncLogRolling/8c4e53ab1a75e944dee323c61b2c115c/.tmp/info/133a107c0f7a43b48852c5248e3955a9
2024-11-17T20:34:51,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/data/default/TestLogRolling-testSlowSyncLogRolling/8c4e53ab1a75e944dee323c61b2c115c/.tmp/info/133a107c0f7a43b48852c5248e3955a9 as hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/data/default/TestLogRolling-testSlowSyncLogRolling/8c4e53ab1a75e944dee323c61b2c115c/info/133a107c0f7a43b48852c5248e3955a9
2024-11-17T20:34:51,453 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/data/default/TestLogRolling-testSlowSyncLogRolling/8c4e53ab1a75e944dee323c61b2c115c/info/133a107c0f7a43b48852c5248e3955a9, entries=7, sequenceid=42, filesize=12.2 K
2024-11-17T20:34:51,454 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 8c4e53ab1a75e944dee323c61b2c115c in 38ms, sequenceid=42, compaction requested=false
2024-11-17T20:34:51,454 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8c4e53ab1a75e944dee323c61b2c115c:

2024-11-17T20:34:51,455 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=39.3 K, sizeToCheck=16.0 K
2024-11-17T20:34:51,455 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-17T20:34:51,455 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/data/default/TestLogRolling-testSlowSyncLogRolling/8c4e53ab1a75e944dee323c61b2c115c/info/b090166d4e784ac58c7abfcd476a394f because midkey is the same as first or last row
2024-11-17T20:34:52,429 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-17T20:34:57,169 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 8c4e53ab1a75e944dee323c61b2c115c, had cached 0 bytes from a total of 40219
2024-11-17T20:34:59,438 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-17T20:34:59,439 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-17T20:34:59,439 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-17T20:34:59,448 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T20:34:59,449 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T20:34:59,449 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-17T20:34:59,449 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-17T20:34:59,449 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=201166883, stopped=false
2024-11-17T20:34:59,450 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=0b0b0ae8de03,35281,1731875603390
2024-11-17T20:34:59,511 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34589-0x1014a807bd70001, quorum=127.0.0.1:51492, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-17T20:34:59,511 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35281-0x1014a807bd70000, quorum=127.0.0.1:51492, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-17T20:34:59,511 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34589-0x1014a807bd70001, quorum=127.0.0.1:51492, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:34:59,511 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35281-0x1014a807bd70000, quorum=127.0.0.1:51492, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:34:59,511 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-17T20:34:59,512 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-17T20:34:59,512 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-17T20:34:59,512 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T20:34:59,513 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35281-0x1014a807bd70000, quorum=127.0.0.1:51492, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-17T20:34:59,513 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34589-0x1014a807bd70001, quorum=127.0.0.1:51492, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-17T20:34:59,513 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '0b0b0ae8de03,34589,1731875604293' *****
2024-11-17T20:34:59,513 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-17T20:34:59,514 INFO  [RS:0;0b0b0ae8de03:34589 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-17T20:34:59,514 INFO  [RS:0;0b0b0ae8de03:34589 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-17T20:34:59,514 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-17T20:34:59,515 INFO  [RS:0;0b0b0ae8de03:34589 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-17T20:34:59,515 INFO  [RS:0;0b0b0ae8de03:34589 {}] regionserver.HRegionServer(3091): Received CLOSE for 8c4e53ab1a75e944dee323c61b2c115c
2024-11-17T20:34:59,516 INFO  [RS:0;0b0b0ae8de03:34589 {}] regionserver.HRegionServer(959): stopping server 0b0b0ae8de03,34589,1731875604293
2024-11-17T20:34:59,516 INFO  [RS:0;0b0b0ae8de03:34589 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-17T20:34:59,516 INFO  [RS:0;0b0b0ae8de03:34589 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;0b0b0ae8de03:34589.
2024-11-17T20:34:59,516 DEBUG [RS:0;0b0b0ae8de03:34589 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-17T20:34:59,516 DEBUG [RS:0;0b0b0ae8de03:34589 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T20:34:59,517 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8c4e53ab1a75e944dee323c61b2c115c, disabling compactions & flushes
2024-11-17T20:34:59,517 INFO  [RS:0;0b0b0ae8de03:34589 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-17T20:34:59,517 INFO  [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testSlowSyncLogRolling,,1731875606717.8c4e53ab1a75e944dee323c61b2c115c.
2024-11-17T20:34:59,517 INFO  [RS:0;0b0b0ae8de03:34589 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-17T20:34:59,517 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testSlowSyncLogRolling,,1731875606717.8c4e53ab1a75e944dee323c61b2c115c.
2024-11-17T20:34:59,517 INFO  [RS:0;0b0b0ae8de03:34589 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-17T20:34:59,517 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1731875606717.8c4e53ab1a75e944dee323c61b2c115c. after waiting 0 ms
2024-11-17T20:34:59,517 INFO  [RS:0;0b0b0ae8de03:34589 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-17T20:34:59,517 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1731875606717.8c4e53ab1a75e944dee323c61b2c115c.
2024-11-17T20:34:59,518 INFO  [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 8c4e53ab1a75e944dee323c61b2c115c 1/1 column families, dataSize=3.15 KB heapSize=3.63 KB
2024-11-17T20:34:59,518 INFO  [RS:0;0b0b0ae8de03:34589 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-11-17T20:34:59,518 DEBUG [RS:0;0b0b0ae8de03:34589 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 8c4e53ab1a75e944dee323c61b2c115c=TestLogRolling-testSlowSyncLogRolling,,1731875606717.8c4e53ab1a75e944dee323c61b2c115c.}
2024-11-17T20:34:59,518 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-17T20:34:59,518 INFO  [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-17T20:34:59,519 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-17T20:34:59,519 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-17T20:34:59,519 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-17T20:34:59,519 DEBUG [RS:0;0b0b0ae8de03:34589 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 8c4e53ab1a75e944dee323c61b2c115c
2024-11-17T20:34:59,519 INFO  [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.65 KB heapSize=3.67 KB
2024-11-17T20:34:59,524 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/data/default/TestLogRolling-testSlowSyncLogRolling/8c4e53ab1a75e944dee323c61b2c115c/.tmp/info/544dd6b70b1d408b9cea34deb1835fa1 is 1080, key is row0029/info:/1731875693421/Put/seqid=0
2024-11-17T20:34:59,531 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741850_1026 (size=8193)
2024-11-17T20:34:59,531 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46521 is added to blk_1073741850_1026 (size=8193)
2024-11-17T20:34:59,531 INFO  [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.15 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/data/default/TestLogRolling-testSlowSyncLogRolling/8c4e53ab1a75e944dee323c61b2c115c/.tmp/info/544dd6b70b1d408b9cea34deb1835fa1
2024-11-17T20:34:59,540 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/data/default/TestLogRolling-testSlowSyncLogRolling/8c4e53ab1a75e944dee323c61b2c115c/.tmp/info/544dd6b70b1d408b9cea34deb1835fa1 as hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/data/default/TestLogRolling-testSlowSyncLogRolling/8c4e53ab1a75e944dee323c61b2c115c/info/544dd6b70b1d408b9cea34deb1835fa1
2024-11-17T20:34:59,545 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/data/hbase/meta/1588230740/.tmp/info/b25421d71be94f0ab0bb819a56a835b8 is 195, key is TestLogRolling-testSlowSyncLogRolling,,1731875606717.8c4e53ab1a75e944dee323c61b2c115c./info:regioninfo/1731875607192/Put/seqid=0
2024-11-17T20:34:59,549 INFO  [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/data/default/TestLogRolling-testSlowSyncLogRolling/8c4e53ab1a75e944dee323c61b2c115c/info/544dd6b70b1d408b9cea34deb1835fa1, entries=3, sequenceid=48, filesize=8.0 K
2024-11-17T20:34:59,551 INFO  [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 8c4e53ab1a75e944dee323c61b2c115c in 34ms, sequenceid=48, compaction requested=true
2024-11-17T20:34:59,552 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731875606717.8c4e53ab1a75e944dee323c61b2c115c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/data/default/TestLogRolling-testSlowSyncLogRolling/8c4e53ab1a75e944dee323c61b2c115c/info/37b64bf44cf0427387fde3869a974442, hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/data/default/TestLogRolling-testSlowSyncLogRolling/8c4e53ab1a75e944dee323c61b2c115c/info/8bfc6398497b44f79d4a884990d02724, hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/data/default/TestLogRolling-testSlowSyncLogRolling/8c4e53ab1a75e944dee323c61b2c115c/info/f61d5fdc21d044f2a34befa80baaf612] to archive
2024-11-17T20:34:59,554 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741851_1027 (size=7016)
2024-11-17T20:34:59,554 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46521 is added to blk_1073741851_1027 (size=7016)
2024-11-17T20:34:59,555 INFO  [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.45 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/data/hbase/meta/1588230740/.tmp/info/b25421d71be94f0ab0bb819a56a835b8
2024-11-17T20:34:59,556 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731875606717.8c4e53ab1a75e944dee323c61b2c115c.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-17T20:34:59,559 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731875606717.8c4e53ab1a75e944dee323c61b2c115c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/data/default/TestLogRolling-testSlowSyncLogRolling/8c4e53ab1a75e944dee323c61b2c115c/info/37b64bf44cf0427387fde3869a974442 to hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/archive/data/default/TestLogRolling-testSlowSyncLogRolling/8c4e53ab1a75e944dee323c61b2c115c/info/37b64bf44cf0427387fde3869a974442
2024-11-17T20:34:59,561 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731875606717.8c4e53ab1a75e944dee323c61b2c115c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/data/default/TestLogRolling-testSlowSyncLogRolling/8c4e53ab1a75e944dee323c61b2c115c/info/8bfc6398497b44f79d4a884990d02724 to hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/archive/data/default/TestLogRolling-testSlowSyncLogRolling/8c4e53ab1a75e944dee323c61b2c115c/info/8bfc6398497b44f79d4a884990d02724
2024-11-17T20:34:59,563 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731875606717.8c4e53ab1a75e944dee323c61b2c115c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/data/default/TestLogRolling-testSlowSyncLogRolling/8c4e53ab1a75e944dee323c61b2c115c/info/f61d5fdc21d044f2a34befa80baaf612 to hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/archive/data/default/TestLogRolling-testSlowSyncLogRolling/8c4e53ab1a75e944dee323c61b2c115c/info/f61d5fdc21d044f2a34befa80baaf612
2024-11-17T20:34:59,574 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731875606717.8c4e53ab1a75e944dee323c61b2c115c.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=0b0b0ae8de03:35281 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 16 more
2024-11-17T20:34:59,578 WARN  [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731875606717.8c4e53ab1a75e944dee323c61b2c115c.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [37b64bf44cf0427387fde3869a974442=12509, 8bfc6398497b44f79d4a884990d02724=12509, f61d5fdc21d044f2a34befa80baaf612=12509]
2024-11-17T20:34:59,580 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/data/hbase/meta/1588230740/.tmp/ns/ea0afbc58ec4447ca8b5e300fa4109f1 is 43, key is default/ns:d/1731875606410/Put/seqid=0
2024-11-17T20:34:59,584 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/data/default/TestLogRolling-testSlowSyncLogRolling/8c4e53ab1a75e944dee323c61b2c115c/recovered.edits/51.seqid, newMaxSeqId=51, maxSeqId=1
2024-11-17T20:34:59,586 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46521 is added to blk_1073741852_1028 (size=5153)
2024-11-17T20:34:59,586 INFO  [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testSlowSyncLogRolling,,1731875606717.8c4e53ab1a75e944dee323c61b2c115c.
2024-11-17T20:34:59,587 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8c4e53ab1a75e944dee323c61b2c115c:
Waiting for close lock at 1731875699516Running coprocessor pre-close hooks at 1731875699517 (+1 ms)Disabling compacts and flushes for region at 1731875699517Disabling writes for close at 1731875699517Obtaining lock to block concurrent updates at 1731875699518 (+1 ms)Preparing flush snapshotting stores in 8c4e53ab1a75e944dee323c61b2c115c at 1731875699518Finished memstore snapshotting TestLogRolling-testSlowSyncLogRolling,,1731875606717.8c4e53ab1a75e944dee323c61b2c115c., syncing WAL and waiting on mvcc, flushsize=dataSize=3228, getHeapSize=3696, getOffHeapSize=0, getCellsCount=3 at 1731875699518Flushing stores of TestLogRolling-testSlowSyncLogRolling,,1731875606717.8c4e53ab1a75e944dee323c61b2c115c. at 1731875699520 (+2 ms)Flushing 8c4e53ab1a75e944dee323c61b2c115c/info: creating writer at 1731875699520Flushing 8c4e53ab1a75e944dee323c61b2c115c/info: appending metadata at 1731875699524 (+4 ms)Flushing 8c4e53ab1a75e944dee323c61b2c115c/info: closing flushed file at 1731875699524Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@21bc8b7e: reopening flushed file at 1731875699539 (+15 ms)Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 8c4e53ab1a75e944dee323c61b2c115c in 34ms, sequenceid=48, compaction requested=true at 1731875699551 (+12 ms)Writing region close event to WAL at 1731875699579 (+28 ms)Running coprocessor post-close hooks at 1731875699585 (+6 ms)Closed at 1731875699586 (+1 ms)
2024-11-17T20:34:59,587 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testSlowSyncLogRolling,,1731875606717.8c4e53ab1a75e944dee323c61b2c115c.
2024-11-17T20:34:59,587 INFO  [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/data/hbase/meta/1588230740/.tmp/ns/ea0afbc58ec4447ca8b5e300fa4109f1
2024-11-17T20:34:59,587 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741852_1028 (size=5153)
2024-11-17T20:34:59,611 INFO  [regionserver/0b0b0ae8de03:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-17T20:34:59,611 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/data/hbase/meta/1588230740/.tmp/table/5e8bd147f747466cb0b707605a1864f6 is 73, key is TestLogRolling-testSlowSyncLogRolling/table:state/1731875607208/Put/seqid=0
2024-11-17T20:34:59,617 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741853_1029 (size=5396)
2024-11-17T20:34:59,617 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46521 is added to blk_1073741853_1029 (size=5396)
2024-11-17T20:34:59,618 INFO  [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=138 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/data/hbase/meta/1588230740/.tmp/table/5e8bd147f747466cb0b707605a1864f6
2024-11-17T20:34:59,627 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/data/hbase/meta/1588230740/.tmp/info/b25421d71be94f0ab0bb819a56a835b8 as hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/data/hbase/meta/1588230740/info/b25421d71be94f0ab0bb819a56a835b8
2024-11-17T20:34:59,636 INFO  [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/data/hbase/meta/1588230740/info/b25421d71be94f0ab0bb819a56a835b8, entries=10, sequenceid=11, filesize=6.9 K
2024-11-17T20:34:59,637 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/data/hbase/meta/1588230740/.tmp/ns/ea0afbc58ec4447ca8b5e300fa4109f1 as hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/data/hbase/meta/1588230740/ns/ea0afbc58ec4447ca8b5e300fa4109f1
2024-11-17T20:34:59,647 INFO  [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/data/hbase/meta/1588230740/ns/ea0afbc58ec4447ca8b5e300fa4109f1, entries=2, sequenceid=11, filesize=5.0 K
2024-11-17T20:34:59,649 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/data/hbase/meta/1588230740/.tmp/table/5e8bd147f747466cb0b707605a1864f6 as hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/data/hbase/meta/1588230740/table/5e8bd147f747466cb0b707605a1864f6
2024-11-17T20:34:59,658 INFO  [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/data/hbase/meta/1588230740/table/5e8bd147f747466cb0b707605a1864f6, entries=2, sequenceid=11, filesize=5.3 K
2024-11-17T20:34:59,659 INFO  [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.65 KB/1692, heapSize ~3.38 KB/3456, currentSize=0 B/0 for 1588230740 in 140ms, sequenceid=11, compaction requested=false
2024-11-17T20:34:59,664 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-11-17T20:34:59,665 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-17T20:34:59,665 INFO  [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-17T20:34:59,665 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731875699518Running coprocessor pre-close hooks at 1731875699518Disabling compacts and flushes for region at 1731875699518Disabling writes for close at 1731875699519 (+1 ms)Obtaining lock to block concurrent updates at 1731875699519Preparing flush snapshotting stores in 1588230740 at 1731875699519Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1692, getHeapSize=3696, getOffHeapSize=0, getCellsCount=14 at 1731875699520 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731875699521 (+1 ms)Flushing 1588230740/info: creating writer at 1731875699521Flushing 1588230740/info: appending metadata at 1731875699545 (+24 ms)Flushing 1588230740/info: closing flushed file at 1731875699545Flushing 1588230740/ns: creating writer at 1731875699563 (+18 ms)Flushing 1588230740/ns: appending metadata at 1731875699580 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1731875699580Flushing 1588230740/table: creating writer at 1731875699595 (+15 ms)Flushing 1588230740/table: appending metadata at 1731875699611 (+16 ms)Flushing 1588230740/table: closing flushed file at 1731875699611Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@47a43e76: reopening flushed file at 1731875699626 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@41c7c087: reopening flushed file at 1731875699636 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@33363b3a: reopening flushed file at 1731875699647 (+11 ms)Finished flush of dataSize ~1.65 KB/1692, heapSize ~3.38 KB/3456, currentSize=0 B/0 for 1588230740 in 140ms, sequenceid=11, compaction requested=false at 1731875699659 (+12 ms)Writing region close event to WAL at 1731875699660 (+1 ms)Running coprocessor post-close hooks at 1731875699665 (+5 ms)Closed at 1731875699665
2024-11-17T20:34:59,665 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-17T20:34:59,673 INFO  [regionserver/0b0b0ae8de03:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-11-17T20:34:59,674 INFO  [regionserver/0b0b0ae8de03:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-11-17T20:34:59,719 INFO  [RS:0;0b0b0ae8de03:34589 {}] regionserver.HRegionServer(976): stopping server 0b0b0ae8de03,34589,1731875604293; all regions closed.
2024-11-17T20:34:59,721 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:34:59,721 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:34:59,722 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:34:59,722 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:34:59,722 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:34:59,728 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46521 is added to blk_1073741834_1010 (size=3066)
2024-11-17T20:34:59,729 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741834_1010 (size=3066)
2024-11-17T20:34:59,734 DEBUG [RS:0;0b0b0ae8de03:34589 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/oldWALs
2024-11-17T20:34:59,734 INFO  [RS:0;0b0b0ae8de03:34589 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 0b0b0ae8de03%2C34589%2C1731875604293.meta:.meta(num 1731875606243)
2024-11-17T20:34:59,734 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:34:59,734 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:34:59,734 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:34:59,734 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:34:59,735 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:34:59,736 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46521 is added to blk_1073741848_1024 (size=12695)
2024-11-17T20:34:59,737 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741848_1024 (size=12695)
2024-11-17T20:34:59,740 DEBUG [RS:0;0b0b0ae8de03:34589 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/oldWALs
2024-11-17T20:34:59,740 INFO  [RS:0;0b0b0ae8de03:34589 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 0b0b0ae8de03%2C34589%2C1731875604293:(num 1731875679384)
2024-11-17T20:34:59,740 DEBUG [RS:0;0b0b0ae8de03:34589 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T20:34:59,741 INFO  [RS:0;0b0b0ae8de03:34589 {}] regionserver.LeaseManager(133): Closed leases
2024-11-17T20:34:59,741 INFO  [RS:0;0b0b0ae8de03:34589 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-17T20:34:59,741 INFO  [RS:0;0b0b0ae8de03:34589 {}] hbase.ChoreService(370): Chore service for: regionserver/0b0b0ae8de03:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-17T20:34:59,741 INFO  [RS:0;0b0b0ae8de03:34589 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-17T20:34:59,741 INFO  [regionserver/0b0b0ae8de03:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-17T20:34:59,741 INFO  [RS:0;0b0b0ae8de03:34589 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34589
2024-11-17T20:34:59,763 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35281-0x1014a807bd70000, quorum=127.0.0.1:51492, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-17T20:34:59,763 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34589-0x1014a807bd70001, quorum=127.0.0.1:51492, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/0b0b0ae8de03,34589,1731875604293
2024-11-17T20:34:59,763 INFO  [RS:0;0b0b0ae8de03:34589 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-17T20:34:59,774 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [0b0b0ae8de03,34589,1731875604293]
2024-11-17T20:34:59,784 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/0b0b0ae8de03,34589,1731875604293 already deleted, retry=false
2024-11-17T20:34:59,785 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 0b0b0ae8de03,34589,1731875604293 expired; onlineServers=0
2024-11-17T20:34:59,785 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '0b0b0ae8de03,35281,1731875603390' *****
2024-11-17T20:34:59,785 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-17T20:34:59,785 INFO  [M:0;0b0b0ae8de03:35281 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-17T20:34:59,785 INFO  [M:0;0b0b0ae8de03:35281 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-17T20:34:59,786 DEBUG [M:0;0b0b0ae8de03:35281 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-17T20:34:59,786 DEBUG [M:0;0b0b0ae8de03:35281 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-17T20:34:59,786 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-17T20:34:59,786 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster-HFileCleaner.small.0-1731875605466 {}] cleaner.HFileCleaner(306): Exit Thread[master/0b0b0ae8de03:0:becomeActiveMaster-HFileCleaner.small.0-1731875605466,5,FailOnTimeoutGroup]
2024-11-17T20:34:59,786 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster-HFileCleaner.large.0-1731875605465 {}] cleaner.HFileCleaner(306): Exit Thread[master/0b0b0ae8de03:0:becomeActiveMaster-HFileCleaner.large.0-1731875605465,5,FailOnTimeoutGroup]
2024-11-17T20:34:59,787 INFO  [M:0;0b0b0ae8de03:35281 {}] hbase.ChoreService(370): Chore service for: master/0b0b0ae8de03:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-17T20:34:59,787 INFO  [M:0;0b0b0ae8de03:35281 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-17T20:34:59,787 DEBUG [M:0;0b0b0ae8de03:35281 {}] master.HMaster(1795): Stopping service threads
2024-11-17T20:34:59,787 INFO  [M:0;0b0b0ae8de03:35281 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-17T20:34:59,787 INFO  [M:0;0b0b0ae8de03:35281 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-17T20:34:59,789 INFO  [M:0;0b0b0ae8de03:35281 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-17T20:34:59,789 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-17T20:34:59,794 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35281-0x1014a807bd70000, quorum=127.0.0.1:51492, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-17T20:34:59,794 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35281-0x1014a807bd70000, quorum=127.0.0.1:51492, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:34:59,795 DEBUG [M:0;0b0b0ae8de03:35281 {}] zookeeper.ZKUtil(347): master:35281-0x1014a807bd70000, quorum=127.0.0.1:51492, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-17T20:34:59,795 WARN  [M:0;0b0b0ae8de03:35281 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-17T20:34:59,795 INFO  [M:0;0b0b0ae8de03:35281 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/.lastflushedseqids
2024-11-17T20:34:59,806 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46521 is added to blk_1073741854_1030 (size=130)
2024-11-17T20:34:59,806 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741854_1030 (size=130)
2024-11-17T20:34:59,806 INFO  [M:0;0b0b0ae8de03:35281 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-17T20:34:59,807 INFO  [M:0;0b0b0ae8de03:35281 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-17T20:34:59,807 DEBUG [M:0;0b0b0ae8de03:35281 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-17T20:34:59,807 INFO  [M:0;0b0b0ae8de03:35281 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T20:34:59,807 DEBUG [M:0;0b0b0ae8de03:35281 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T20:34:59,807 DEBUG [M:0;0b0b0ae8de03:35281 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-17T20:34:59,807 DEBUG [M:0;0b0b0ae8de03:35281 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T20:34:59,807 INFO  [M:0;0b0b0ae8de03:35281 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.02 KB heapSize=29.20 KB
2024-11-17T20:34:59,825 DEBUG [M:0;0b0b0ae8de03:35281 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/74f867f111b540b6be28178c25511664 is 82, key is hbase:meta,,1/info:regioninfo/1731875606318/Put/seqid=0
2024-11-17T20:34:59,831 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741855_1031 (size=5672)
2024-11-17T20:34:59,832 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46521 is added to blk_1073741855_1031 (size=5672)
2024-11-17T20:34:59,832 INFO  [M:0;0b0b0ae8de03:35281 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/74f867f111b540b6be28178c25511664
2024-11-17T20:34:59,853 DEBUG [M:0;0b0b0ae8de03:35281 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ce3e9a4ccb934bef9867046558d4a711 is 766, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731875607216/Put/seqid=0
2024-11-17T20:34:59,860 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741856_1032 (size=6247)
2024-11-17T20:34:59,860 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46521 is added to blk_1073741856_1032 (size=6247)
2024-11-17T20:34:59,861 INFO  [M:0;0b0b0ae8de03:35281 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.42 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ce3e9a4ccb934bef9867046558d4a711
2024-11-17T20:34:59,867 INFO  [M:0;0b0b0ae8de03:35281 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ce3e9a4ccb934bef9867046558d4a711
2024-11-17T20:34:59,874 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34589-0x1014a807bd70001, quorum=127.0.0.1:51492, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-17T20:34:59,874 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34589-0x1014a807bd70001, quorum=127.0.0.1:51492, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-17T20:34:59,875 INFO  [RS:0;0b0b0ae8de03:34589 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-17T20:34:59,875 INFO  [RS:0;0b0b0ae8de03:34589 {}] regionserver.HRegionServer(1031): Exiting; stopping=0b0b0ae8de03,34589,1731875604293; zookeeper connection closed.
2024-11-17T20:34:59,875 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7d7e894d {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7d7e894d
2024-11-17T20:34:59,875 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-17T20:34:59,882 DEBUG [M:0;0b0b0ae8de03:35281 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b93a659c43864b5ea55eddacd92078e7 is 69, key is 0b0b0ae8de03,34589,1731875604293/rs:state/1731875605529/Put/seqid=0
2024-11-17T20:34:59,888 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741857_1033 (size=5156)
2024-11-17T20:34:59,889 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46521 is added to blk_1073741857_1033 (size=5156)
2024-11-17T20:34:59,889 INFO  [M:0;0b0b0ae8de03:35281 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b93a659c43864b5ea55eddacd92078e7
2024-11-17T20:34:59,922 DEBUG [M:0;0b0b0ae8de03:35281 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/0799d5c2d1ca4fe981318c808ccd1236 is 52, key is load_balancer_on/state:d/1731875606697/Put/seqid=0
2024-11-17T20:34:59,930 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741858_1034 (size=5056)
2024-11-17T20:34:59,930 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46521 is added to blk_1073741858_1034 (size=5056)
2024-11-17T20:34:59,931 INFO  [M:0;0b0b0ae8de03:35281 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/0799d5c2d1ca4fe981318c808ccd1236
2024-11-17T20:34:59,939 DEBUG [M:0;0b0b0ae8de03:35281 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/74f867f111b540b6be28178c25511664 as hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/74f867f111b540b6be28178c25511664
2024-11-17T20:34:59,946 INFO  [M:0;0b0b0ae8de03:35281 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/74f867f111b540b6be28178c25511664, entries=8, sequenceid=59, filesize=5.5 K
2024-11-17T20:34:59,948 DEBUG [M:0;0b0b0ae8de03:35281 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ce3e9a4ccb934bef9867046558d4a711 as hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/ce3e9a4ccb934bef9867046558d4a711
2024-11-17T20:34:59,955 INFO  [M:0;0b0b0ae8de03:35281 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ce3e9a4ccb934bef9867046558d4a711
2024-11-17T20:34:59,955 INFO  [M:0;0b0b0ae8de03:35281 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/ce3e9a4ccb934bef9867046558d4a711, entries=6, sequenceid=59, filesize=6.1 K
2024-11-17T20:34:59,956 DEBUG [M:0;0b0b0ae8de03:35281 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b93a659c43864b5ea55eddacd92078e7 as hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/b93a659c43864b5ea55eddacd92078e7
2024-11-17T20:34:59,963 INFO  [M:0;0b0b0ae8de03:35281 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/b93a659c43864b5ea55eddacd92078e7, entries=1, sequenceid=59, filesize=5.0 K
2024-11-17T20:34:59,965 DEBUG [M:0;0b0b0ae8de03:35281 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/0799d5c2d1ca4fe981318c808ccd1236 as hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/0799d5c2d1ca4fe981318c808ccd1236
2024-11-17T20:34:59,972 INFO  [M:0;0b0b0ae8de03:35281 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/0799d5c2d1ca4fe981318c808ccd1236, entries=1, sequenceid=59, filesize=4.9 K
2024-11-17T20:34:59,974 INFO  [M:0;0b0b0ae8de03:35281 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.02 KB/23576, heapSize ~29.14 KB/29840, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 167ms, sequenceid=59, compaction requested=false
2024-11-17T20:34:59,975 INFO  [M:0;0b0b0ae8de03:35281 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T20:34:59,976 DEBUG [M:0;0b0b0ae8de03:35281 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731875699807Disabling compacts and flushes for region at 1731875699807Disabling writes for close at 1731875699807Obtaining lock to block concurrent updates at 1731875699807Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731875699807Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23576, getHeapSize=29840, getOffHeapSize=0, getCellsCount=70 at 1731875699808 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731875699809 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731875699809Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731875699825 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731875699825Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731875699837 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731875699852 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731875699852Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731875699867 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731875699881 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731875699881Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731875699901 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731875699921 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731875699921Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@135bdbf3: reopening flushed file at 1731875699938 (+17 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@bfc0819: reopening flushed file at 1731875699946 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1d7d7dc: reopening flushed file at 1731875699955 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@40e3854d: reopening flushed file at 1731875699964 (+9 ms)Finished flush of dataSize ~23.02 KB/23576, heapSize ~29.14 KB/29840, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 167ms, sequenceid=59, compaction requested=false at 1731875699974 (+10 ms)Writing region close event to WAL at 1731875699975 (+1 ms)Closed at 1731875699975
2024-11-17T20:34:59,976 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:34:59,977 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:34:59,977 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:34:59,977 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:34:59,977 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:34:59,979 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741830_1006 (size=27973)
2024-11-17T20:34:59,979 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46521 is added to blk_1073741830_1006 (size=27973)
2024-11-17T20:34:59,980 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-17T20:34:59,980 INFO  [M:0;0b0b0ae8de03:35281 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-17T20:34:59,981 INFO  [M:0;0b0b0ae8de03:35281 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35281
2024-11-17T20:34:59,981 INFO  [M:0;0b0b0ae8de03:35281 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-17T20:35:00,091 INFO  [M:0;0b0b0ae8de03:35281 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-17T20:35:00,091 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35281-0x1014a807bd70000, quorum=127.0.0.1:51492, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-17T20:35:00,091 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35281-0x1014a807bd70000, quorum=127.0.0.1:51492, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-17T20:35:00,132 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1bf97579{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T20:35:00,135 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@22b88bcb{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-17T20:35:00,135 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-17T20:35:00,135 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2d48d695{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-17T20:35:00,135 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@11effdcd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d5cf336-d7e3-9af4-11bd-2d3b3572aece/hadoop.log.dir/,STOPPED}
2024-11-17T20:35:00,139 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-17T20:35:00,139 WARN  [BP-1663635380-172.17.0.2-1731875598538 heartbeating to localhost/127.0.0.1:35585 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-17T20:35:00,139 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-17T20:35:00,139 WARN  [BP-1663635380-172.17.0.2-1731875598538 heartbeating to localhost/127.0.0.1:35585 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1663635380-172.17.0.2-1731875598538 (Datanode Uuid 93d16be3-7334-46a2-96dc-9dfa33e1fa4c) service to localhost/127.0.0.1:35585
2024-11-17T20:35:00,140 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d5cf336-d7e3-9af4-11bd-2d3b3572aece/cluster_736ee10e-3507-4deb-8e28-689799b59293/data/data3/current/BP-1663635380-172.17.0.2-1731875598538 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T20:35:00,141 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d5cf336-d7e3-9af4-11bd-2d3b3572aece/cluster_736ee10e-3507-4deb-8e28-689799b59293/data/data4/current/BP-1663635380-172.17.0.2-1731875598538 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T20:35:00,141 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-17T20:35:00,144 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7b07d1ba{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T20:35:00,144 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@43e0a762{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-17T20:35:00,145 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-17T20:35:00,145 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@371e191c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-17T20:35:00,145 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@28778f0f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d5cf336-d7e3-9af4-11bd-2d3b3572aece/hadoop.log.dir/,STOPPED}
2024-11-17T20:35:00,146 WARN  [BP-1663635380-172.17.0.2-1731875598538 heartbeating to localhost/127.0.0.1:35585 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-17T20:35:00,146 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-17T20:35:00,146 WARN  [BP-1663635380-172.17.0.2-1731875598538 heartbeating to localhost/127.0.0.1:35585 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1663635380-172.17.0.2-1731875598538 (Datanode Uuid d62433fe-f6f4-4dd0-865c-90eff6d6a8d3) service to localhost/127.0.0.1:35585
2024-11-17T20:35:00,146 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-17T20:35:00,147 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d5cf336-d7e3-9af4-11bd-2d3b3572aece/cluster_736ee10e-3507-4deb-8e28-689799b59293/data/data1/current/BP-1663635380-172.17.0.2-1731875598538 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T20:35:00,147 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d5cf336-d7e3-9af4-11bd-2d3b3572aece/cluster_736ee10e-3507-4deb-8e28-689799b59293/data/data2/current/BP-1663635380-172.17.0.2-1731875598538 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T20:35:00,147 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-17T20:35:00,155 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@735fa16a{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-17T20:35:00,155 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6c26a5a3{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-17T20:35:00,155 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-17T20:35:00,155 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@70be1389{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-17T20:35:00,156 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@ddc8467{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d5cf336-d7e3-9af4-11bd-2d3b3572aece/hadoop.log.dir/,STOPPED}
2024-11-17T20:35:00,164 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-17T20:35:00,195 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-17T20:35:00,204 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testSlowSyncLogRolling Thread=81 (was 12)
Potentially hanging thread: RPCClient-NioEventLoopGroup-4-1
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: sync.4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-5-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:35585 from jenkins.hfs.0
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: sync.1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-5-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-3-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-1-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: nioEventLoopGroup-4-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Async-Client-Retry-Timer-pool-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-7
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: SSL Certificates Store Monitor
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.Object.wait(Object.java:338)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:537)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176)
	app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: regionserver/0b0b0ae8de03:0.procedureResultReporter
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:35585
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: org.apache.hadoop.hdfs.PeerCache@2aa892a4
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:253)
	app//org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46)
	app//org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-5-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: SessionTracker
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163)

Potentially hanging thread: SnapshotHandlerChoreCleaner
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.0@localhost:35585
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-1-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Timer for 'HBase' metrics system
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: Monitor thread for TaskMonitor
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:35585
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: master/0b0b0ae8de03:0:becomeActiveMaster-MemStoreChunkPool Statistics
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HBase-Metrics2-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-2-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-5
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: master/0b0b0ae8de03:0:becomeActiveMaster-MemStoreChunkPool Statistics
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: ForkJoinPool-2-worker-4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: Time-limited test.named-queue-events-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47)
	app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56)
	app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159)
	app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-2-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: GcTimeMonitor obsWindow = 60000, sleepInterval = 5000, maxGcTimePerc = 100
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.util.GcTimeMonitor.run(GcTimeMonitor.java:161)

Potentially hanging thread: nioEventLoopGroup-4-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-6
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-3-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-3-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-4-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:35585 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: sync.2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: sync.4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:35585 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: ForkJoinPool-2-worker-2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: RpcClient-timer-pool-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-2-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:35585
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-3
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:35585
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: weak-ref-cleaner-strictcontextstorage
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176)
	app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: HMaster-EventLoopGroup-1-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: region-location-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=404 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=176 (was 251), ProcessCount=11 (was 11), AvailableMemoryMB=8632 (was 8887)
2024-11-17T20:35:00,210 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnDatanodeDeath Thread=82, OpenFileDescriptor=404, MaxFileDescriptor=1048576, SystemLoadAverage=176, ProcessCount=11, AvailableMemoryMB=8632
2024-11-17T20:35:00,210 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-17T20:35:00,210 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d5cf336-d7e3-9af4-11bd-2d3b3572aece/hadoop.log.dir so I do NOT create it in target/test-data/e2e45e89-d349-682c-1566-7129f4ac0660
2024-11-17T20:35:00,210 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d5cf336-d7e3-9af4-11bd-2d3b3572aece/hadoop.tmp.dir so I do NOT create it in target/test-data/e2e45e89-d349-682c-1566-7129f4ac0660
2024-11-17T20:35:00,210 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2e45e89-d349-682c-1566-7129f4ac0660/cluster_0dce1bfb-1fca-ebc8-fb54-79ce3a8be4db, deleteOnExit=true
2024-11-17T20:35:00,210 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-17T20:35:00,211 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2e45e89-d349-682c-1566-7129f4ac0660/test.cache.data in system properties and HBase conf
2024-11-17T20:35:00,211 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2e45e89-d349-682c-1566-7129f4ac0660/hadoop.tmp.dir in system properties and HBase conf
2024-11-17T20:35:00,211 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2e45e89-d349-682c-1566-7129f4ac0660/hadoop.log.dir in system properties and HBase conf
2024-11-17T20:35:00,211 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2e45e89-d349-682c-1566-7129f4ac0660/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-17T20:35:00,211 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2e45e89-d349-682c-1566-7129f4ac0660/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-17T20:35:00,211 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-17T20:35:00,211 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-17T20:35:00,211 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2e45e89-d349-682c-1566-7129f4ac0660/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-17T20:35:00,211 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2e45e89-d349-682c-1566-7129f4ac0660/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-17T20:35:00,212 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2e45e89-d349-682c-1566-7129f4ac0660/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-17T20:35:00,212 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2e45e89-d349-682c-1566-7129f4ac0660/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-17T20:35:00,212 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2e45e89-d349-682c-1566-7129f4ac0660/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-17T20:35:00,212 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2e45e89-d349-682c-1566-7129f4ac0660/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-17T20:35:00,212 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2e45e89-d349-682c-1566-7129f4ac0660/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-17T20:35:00,212 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2e45e89-d349-682c-1566-7129f4ac0660/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-17T20:35:00,212 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2e45e89-d349-682c-1566-7129f4ac0660/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-17T20:35:00,212 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2e45e89-d349-682c-1566-7129f4ac0660/nfs.dump.dir in system properties and HBase conf
2024-11-17T20:35:00,212 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2e45e89-d349-682c-1566-7129f4ac0660/java.io.tmpdir in system properties and HBase conf
2024-11-17T20:35:00,212 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2e45e89-d349-682c-1566-7129f4ac0660/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-17T20:35:00,213 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2e45e89-d349-682c-1566-7129f4ac0660/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-17T20:35:00,213 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2e45e89-d349-682c-1566-7129f4ac0660/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-17T20:35:00,226 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-17T20:35:00,590 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-17T20:35:00,596 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-17T20:35:00,597 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-17T20:35:00,597 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-17T20:35:00,597 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-17T20:35:00,598 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-17T20:35:00,598 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2152d149{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2e45e89-d349-682c-1566-7129f4ac0660/hadoop.log.dir/,AVAILABLE}
2024-11-17T20:35:00,599 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@542ee468{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-17T20:35:00,706 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@81550dd{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2e45e89-d349-682c-1566-7129f4ac0660/java.io.tmpdir/jetty-localhost-35589-hadoop-hdfs-3_4_1-tests_jar-_-any-11428335794979986281/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-17T20:35:00,707 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2d71062d{HTTP/1.1, (http/1.1)}{localhost:35589}
2024-11-17T20:35:00,707 INFO  [Time-limited test {}] server.Server(415): Started @103993ms
2024-11-17T20:35:00,719 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-17T20:35:00,984 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-17T20:35:00,989 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-17T20:35:00,990 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-17T20:35:00,990 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-17T20:35:00,990 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-17T20:35:00,991 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@21e00560{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2e45e89-d349-682c-1566-7129f4ac0660/hadoop.log.dir/,AVAILABLE}
2024-11-17T20:35:00,991 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3b01355c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-17T20:35:01,086 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@426dc4fa{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2e45e89-d349-682c-1566-7129f4ac0660/java.io.tmpdir/jetty-localhost-35653-hadoop-hdfs-3_4_1-tests_jar-_-any-3812355141699373117/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T20:35:01,086 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@417f4bff{HTTP/1.1, (http/1.1)}{localhost:35653}
2024-11-17T20:35:01,087 INFO  [Time-limited test {}] server.Server(415): Started @104373ms
2024-11-17T20:35:01,088 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-17T20:35:01,124 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-17T20:35:01,128 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-17T20:35:01,129 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-17T20:35:01,129 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-17T20:35:01,129 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-17T20:35:01,132 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@17d00685{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2e45e89-d349-682c-1566-7129f4ac0660/hadoop.log.dir/,AVAILABLE}
2024-11-17T20:35:01,132 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2220be00{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-17T20:35:01,233 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@210f4ee9{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2e45e89-d349-682c-1566-7129f4ac0660/java.io.tmpdir/jetty-localhost-45273-hadoop-hdfs-3_4_1-tests_jar-_-any-13721941873639870421/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T20:35:01,234 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1dedf320{HTTP/1.1, (http/1.1)}{localhost:45273}
2024-11-17T20:35:01,234 INFO  [Time-limited test {}] server.Server(415): Started @104520ms
2024-11-17T20:35:01,236 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-17T20:35:02,248 WARN  [Thread-452 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2e45e89-d349-682c-1566-7129f4ac0660/cluster_0dce1bfb-1fca-ebc8-fb54-79ce3a8be4db/data/data1/current/BP-1418661057-172.17.0.2-1731875700237/current, will proceed with Du for space computation calculation, 
2024-11-17T20:35:02,248 WARN  [Thread-453 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2e45e89-d349-682c-1566-7129f4ac0660/cluster_0dce1bfb-1fca-ebc8-fb54-79ce3a8be4db/data/data2/current/BP-1418661057-172.17.0.2-1731875700237/current, will proceed with Du for space computation calculation, 
2024-11-17T20:35:02,264 WARN  [Thread-416 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-17T20:35:02,267 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9f1306f0d636b3e0 with lease ID 0x29b6637c7e5beeb4: Processing first storage report for DS-e32d4b1c-cf73-4c36-a366-8e6be0010f48 from datanode DatanodeRegistration(127.0.0.1:38071, datanodeUuid=d7d06dc4-9366-414f-a7e6-256ba7e4b4fc, infoPort=42565, infoSecurePort=0, ipcPort=33605, storageInfo=lv=-57;cid=testClusterID;nsid=1807000045;c=1731875700237)
2024-11-17T20:35:02,267 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9f1306f0d636b3e0 with lease ID 0x29b6637c7e5beeb4: from storage DS-e32d4b1c-cf73-4c36-a366-8e6be0010f48 node DatanodeRegistration(127.0.0.1:38071, datanodeUuid=d7d06dc4-9366-414f-a7e6-256ba7e4b4fc, infoPort=42565, infoSecurePort=0, ipcPort=33605, storageInfo=lv=-57;cid=testClusterID;nsid=1807000045;c=1731875700237), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-17T20:35:02,267 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9f1306f0d636b3e0 with lease ID 0x29b6637c7e5beeb4: Processing first storage report for DS-bf875be2-b00f-4386-9d93-513faf8b8c35 from datanode DatanodeRegistration(127.0.0.1:38071, datanodeUuid=d7d06dc4-9366-414f-a7e6-256ba7e4b4fc, infoPort=42565, infoSecurePort=0, ipcPort=33605, storageInfo=lv=-57;cid=testClusterID;nsid=1807000045;c=1731875700237)
2024-11-17T20:35:02,267 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9f1306f0d636b3e0 with lease ID 0x29b6637c7e5beeb4: from storage DS-bf875be2-b00f-4386-9d93-513faf8b8c35 node DatanodeRegistration(127.0.0.1:38071, datanodeUuid=d7d06dc4-9366-414f-a7e6-256ba7e4b4fc, infoPort=42565, infoSecurePort=0, ipcPort=33605, storageInfo=lv=-57;cid=testClusterID;nsid=1807000045;c=1731875700237), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-17T20:35:02,387 WARN  [Thread-464 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2e45e89-d349-682c-1566-7129f4ac0660/cluster_0dce1bfb-1fca-ebc8-fb54-79ce3a8be4db/data/data4/current/BP-1418661057-172.17.0.2-1731875700237/current, will proceed with Du for space computation calculation, 
2024-11-17T20:35:02,387 WARN  [Thread-463 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2e45e89-d349-682c-1566-7129f4ac0660/cluster_0dce1bfb-1fca-ebc8-fb54-79ce3a8be4db/data/data3/current/BP-1418661057-172.17.0.2-1731875700237/current, will proceed with Du for space computation calculation, 
2024-11-17T20:35:02,404 WARN  [Thread-439 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-17T20:35:02,408 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x79e4e584d2f89a0c with lease ID 0x29b6637c7e5beeb5: Processing first storage report for DS-e9b9944c-3dc7-4172-b3c8-040651721799 from datanode DatanodeRegistration(127.0.0.1:41377, datanodeUuid=38a3733d-980e-474e-a4a6-57da3818e57a, infoPort=38843, infoSecurePort=0, ipcPort=43277, storageInfo=lv=-57;cid=testClusterID;nsid=1807000045;c=1731875700237)
2024-11-17T20:35:02,408 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x79e4e584d2f89a0c with lease ID 0x29b6637c7e5beeb5: from storage DS-e9b9944c-3dc7-4172-b3c8-040651721799 node DatanodeRegistration(127.0.0.1:41377, datanodeUuid=38a3733d-980e-474e-a4a6-57da3818e57a, infoPort=38843, infoSecurePort=0, ipcPort=43277, storageInfo=lv=-57;cid=testClusterID;nsid=1807000045;c=1731875700237), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-17T20:35:02,408 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x79e4e584d2f89a0c with lease ID 0x29b6637c7e5beeb5: Processing first storage report for DS-c783a2bb-3982-44eb-8207-a8a51493eb46 from datanode DatanodeRegistration(127.0.0.1:41377, datanodeUuid=38a3733d-980e-474e-a4a6-57da3818e57a, infoPort=38843, infoSecurePort=0, ipcPort=43277, storageInfo=lv=-57;cid=testClusterID;nsid=1807000045;c=1731875700237)
2024-11-17T20:35:02,408 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x79e4e584d2f89a0c with lease ID 0x29b6637c7e5beeb5: from storage DS-c783a2bb-3982-44eb-8207-a8a51493eb46 node DatanodeRegistration(127.0.0.1:41377, datanodeUuid=38a3733d-980e-474e-a4a6-57da3818e57a, infoPort=38843, infoSecurePort=0, ipcPort=43277, storageInfo=lv=-57;cid=testClusterID;nsid=1807000045;c=1731875700237), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-17T20:35:02,483 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2e45e89-d349-682c-1566-7129f4ac0660
2024-11-17T20:35:02,486 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2e45e89-d349-682c-1566-7129f4ac0660/cluster_0dce1bfb-1fca-ebc8-fb54-79ce3a8be4db/zookeeper_0, clientPort=52057, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2e45e89-d349-682c-1566-7129f4ac0660/cluster_0dce1bfb-1fca-ebc8-fb54-79ce3a8be4db/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2e45e89-d349-682c-1566-7129f4ac0660/cluster_0dce1bfb-1fca-ebc8-fb54-79ce3a8be4db/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-17T20:35:02,488 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=52057
2024-11-17T20:35:02,488 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T20:35:02,491 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T20:35:02,502 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41377 is added to blk_1073741825_1001 (size=7)
2024-11-17T20:35:02,503 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741825_1001 (size=7)
2024-11-17T20:35:02,504 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:36611/user/jenkins/test-data/dec63fa1-57fb-a401-e986-0477dde7e446 with version=8
2024-11-17T20:35:02,504 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/hbase-staging
2024-11-17T20:35:02,507 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/0b0b0ae8de03:0 server-side Connection retries=45
2024-11-17T20:35:02,507 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-17T20:35:02,507 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-17T20:35:02,508 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-17T20:35:02,508 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-17T20:35:02,508 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-17T20:35:02,508 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-17T20:35:02,508 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-17T20:35:02,509 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39627
2024-11-17T20:35:02,511 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:39627 connecting to ZooKeeper ensemble=127.0.0.1:52057
2024-11-17T20:35:02,569 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:396270x0, quorum=127.0.0.1:52057, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-17T20:35:02,569 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:39627-0x1014a8202800000 connected
2024-11-17T20:35:02,653 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T20:35:02,655 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T20:35:02,658 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39627-0x1014a8202800000, quorum=127.0.0.1:52057, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-17T20:35:02,658 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:36611/user/jenkins/test-data/dec63fa1-57fb-a401-e986-0477dde7e446, hbase.cluster.distributed=false
2024-11-17T20:35:02,661 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39627-0x1014a8202800000, quorum=127.0.0.1:52057, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-17T20:35:02,661 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39627
2024-11-17T20:35:02,661 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39627
2024-11-17T20:35:02,662 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39627
2024-11-17T20:35:02,662 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39627
2024-11-17T20:35:02,662 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39627
2024-11-17T20:35:02,679 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/0b0b0ae8de03:0 server-side Connection retries=45
2024-11-17T20:35:02,679 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-17T20:35:02,679 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-17T20:35:02,679 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-17T20:35:02,679 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-17T20:35:02,679 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-17T20:35:02,679 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-17T20:35:02,679 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-17T20:35:02,680 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45013
2024-11-17T20:35:02,682 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45013 connecting to ZooKeeper ensemble=127.0.0.1:52057
2024-11-17T20:35:02,682 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T20:35:02,685 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T20:35:02,700 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:450130x0, quorum=127.0.0.1:52057, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-17T20:35:02,701 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45013-0x1014a8202800001, quorum=127.0.0.1:52057, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-17T20:35:02,701 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45013-0x1014a8202800001 connected
2024-11-17T20:35:02,701 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-17T20:35:02,702 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-17T20:35:02,702 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45013-0x1014a8202800001, quorum=127.0.0.1:52057, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-17T20:35:02,704 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45013-0x1014a8202800001, quorum=127.0.0.1:52057, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-17T20:35:02,706 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45013
2024-11-17T20:35:02,709 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45013
2024-11-17T20:35:02,710 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45013
2024-11-17T20:35:02,711 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45013
2024-11-17T20:35:02,711 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45013
2024-11-17T20:35:02,724 DEBUG [M:0;0b0b0ae8de03:39627 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;0b0b0ae8de03:39627
2024-11-17T20:35:02,724 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/0b0b0ae8de03,39627,1731875702507
2024-11-17T20:35:02,737 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39627-0x1014a8202800000, quorum=127.0.0.1:52057, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-17T20:35:02,737 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45013-0x1014a8202800001, quorum=127.0.0.1:52057, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-17T20:35:02,738 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39627-0x1014a8202800000, quorum=127.0.0.1:52057, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/0b0b0ae8de03,39627,1731875702507
2024-11-17T20:35:02,748 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45013-0x1014a8202800001, quorum=127.0.0.1:52057, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-17T20:35:02,748 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39627-0x1014a8202800000, quorum=127.0.0.1:52057, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:35:02,748 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45013-0x1014a8202800001, quorum=127.0.0.1:52057, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:35:02,749 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39627-0x1014a8202800000, quorum=127.0.0.1:52057, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-17T20:35:02,749 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/0b0b0ae8de03,39627,1731875702507 from backup master directory
2024-11-17T20:35:02,758 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45013-0x1014a8202800001, quorum=127.0.0.1:52057, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-17T20:35:02,758 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39627-0x1014a8202800000, quorum=127.0.0.1:52057, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/0b0b0ae8de03,39627,1731875702507
2024-11-17T20:35:02,758 WARN  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-17T20:35:02,758 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39627-0x1014a8202800000, quorum=127.0.0.1:52057, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-17T20:35:02,759 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=0b0b0ae8de03,39627,1731875702507
2024-11-17T20:35:02,765 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:36611/user/jenkins/test-data/dec63fa1-57fb-a401-e986-0477dde7e446/hbase.id] with ID: 19318c1e-d59e-47a4-8776-f4c4db6dc4ae
2024-11-17T20:35:02,765 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:36611/user/jenkins/test-data/dec63fa1-57fb-a401-e986-0477dde7e446/.tmp/hbase.id
2024-11-17T20:35:02,771 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741826_1002 (size=42)
2024-11-17T20:35:02,772 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41377 is added to blk_1073741826_1002 (size=42)
2024-11-17T20:35:02,772 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:36611/user/jenkins/test-data/dec63fa1-57fb-a401-e986-0477dde7e446/.tmp/hbase.id]:[hdfs://localhost:36611/user/jenkins/test-data/dec63fa1-57fb-a401-e986-0477dde7e446/hbase.id]
2024-11-17T20:35:02,788 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T20:35:02,788 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-17T20:35:02,791 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 3ms.
2024-11-17T20:35:02,867 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45013-0x1014a8202800001, quorum=127.0.0.1:52057, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:35:02,867 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39627-0x1014a8202800000, quorum=127.0.0.1:52057, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:35:02,875 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741827_1003 (size=196)
2024-11-17T20:35:02,875 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41377 is added to blk_1073741827_1003 (size=196)
2024-11-17T20:35:02,877 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-17T20:35:02,878 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-17T20:35:02,878 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-17T20:35:02,889 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741828_1004 (size=1189)
2024-11-17T20:35:02,889 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41377 is added to blk_1073741828_1004 (size=1189)
2024-11-17T20:35:02,891 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:36611/user/jenkins/test-data/dec63fa1-57fb-a401-e986-0477dde7e446/MasterData/data/master/store
2024-11-17T20:35:02,901 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741829_1005 (size=34)
2024-11-17T20:35:02,901 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41377 is added to blk_1073741829_1005 (size=34)
2024-11-17T20:35:02,902 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T20:35:02,902 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-17T20:35:02,902 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T20:35:02,902 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T20:35:02,902 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-17T20:35:02,902 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T20:35:02,902 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T20:35:02,903 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731875702902Disabling compacts and flushes for region at 1731875702902Disabling writes for close at 1731875702902Writing region close event to WAL at 1731875702902Closed at 1731875702902
2024-11-17T20:35:02,904 WARN  [master/0b0b0ae8de03:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:36611/user/jenkins/test-data/dec63fa1-57fb-a401-e986-0477dde7e446/MasterData/data/master/store/.initializing
2024-11-17T20:35:02,904 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:36611/user/jenkins/test-data/dec63fa1-57fb-a401-e986-0477dde7e446/MasterData/WALs/0b0b0ae8de03,39627,1731875702507
2024-11-17T20:35:02,907 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0b0b0ae8de03%2C39627%2C1731875702507, suffix=, logDir=hdfs://localhost:36611/user/jenkins/test-data/dec63fa1-57fb-a401-e986-0477dde7e446/MasterData/WALs/0b0b0ae8de03,39627,1731875702507, archiveDir=hdfs://localhost:36611/user/jenkins/test-data/dec63fa1-57fb-a401-e986-0477dde7e446/MasterData/oldWALs, maxLogs=10
2024-11-17T20:35:02,908 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0b0b0ae8de03%2C39627%2C1731875702507.1731875702907
2024-11-17T20:35:02,913 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/dec63fa1-57fb-a401-e986-0477dde7e446/MasterData/WALs/0b0b0ae8de03,39627,1731875702507/0b0b0ae8de03%2C39627%2C1731875702507.1731875702907
2024-11-17T20:35:02,920 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42565:42565),(127.0.0.1/127.0.0.1:38843:38843)]
2024-11-17T20:35:02,921 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-17T20:35:02,921 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T20:35:02,921 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T20:35:02,921 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T20:35:02,923 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-17T20:35:02,925 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-17T20:35:02,925 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:35:02,926 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T20:35:02,926 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-17T20:35:02,928 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-17T20:35:02,928 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:35:02,929 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-17T20:35:02,929 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-17T20:35:02,932 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-17T20:35:02,932 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:35:02,933 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-17T20:35:02,933 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-17T20:35:02,935 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-17T20:35:02,935 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:35:02,936 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-17T20:35:02,936 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T20:35:02,937 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36611/user/jenkins/test-data/dec63fa1-57fb-a401-e986-0477dde7e446/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-17T20:35:02,937 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36611/user/jenkins/test-data/dec63fa1-57fb-a401-e986-0477dde7e446/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-17T20:35:02,939 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T20:35:02,939 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T20:35:02,940 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-17T20:35:02,941 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T20:35:02,945 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36611/user/jenkins/test-data/dec63fa1-57fb-a401-e986-0477dde7e446/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-17T20:35:02,945 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=785086, jitterRate=-0.0017115473747253418}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-17T20:35:02,946 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1731875702921Initializing all the Stores at 1731875702923 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731875702923Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731875702923Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731875702923Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731875702923Cleaning up temporary data from old regions at 1731875702939 (+16 ms)Region opened successfully at 1731875702946 (+7 ms)
2024-11-17T20:35:02,947 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-17T20:35:02,977 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5864c184, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0b0b0ae8de03/172.17.0.2:0
2024-11-17T20:35:02,978 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-17T20:35:02,978 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-17T20:35:02,978 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-17T20:35:02,979 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-17T20:35:02,979 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-17T20:35:02,980 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-17T20:35:02,980 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-17T20:35:02,983 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-17T20:35:02,985 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39627-0x1014a8202800000, quorum=127.0.0.1:52057, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-17T20:35:03,011 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-17T20:35:03,012 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-17T20:35:03,013 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39627-0x1014a8202800000, quorum=127.0.0.1:52057, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-17T20:35:03,021 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-17T20:35:03,022 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-17T20:35:03,023 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39627-0x1014a8202800000, quorum=127.0.0.1:52057, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-17T20:35:03,032 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-17T20:35:03,033 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39627-0x1014a8202800000, quorum=127.0.0.1:52057, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-17T20:35:03,042 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-17T20:35:03,045 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39627-0x1014a8202800000, quorum=127.0.0.1:52057, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-17T20:35:03,057 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-17T20:35:03,068 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45013-0x1014a8202800001, quorum=127.0.0.1:52057, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-17T20:35:03,068 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39627-0x1014a8202800000, quorum=127.0.0.1:52057, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-17T20:35:03,068 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45013-0x1014a8202800001, quorum=127.0.0.1:52057, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:35:03,068 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39627-0x1014a8202800000, quorum=127.0.0.1:52057, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:35:03,069 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=0b0b0ae8de03,39627,1731875702507, sessionid=0x1014a8202800000, setting cluster-up flag (Was=false)
2024-11-17T20:35:03,089 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45013-0x1014a8202800001, quorum=127.0.0.1:52057, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:35:03,089 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39627-0x1014a8202800000, quorum=127.0.0.1:52057, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:35:03,121 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-17T20:35:03,122 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0b0b0ae8de03,39627,1731875702507
2024-11-17T20:35:03,142 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39627-0x1014a8202800000, quorum=127.0.0.1:52057, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:35:03,142 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45013-0x1014a8202800001, quorum=127.0.0.1:52057, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:35:03,174 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-17T20:35:03,175 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0b0b0ae8de03,39627,1731875702507
2024-11-17T20:35:03,177 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:36611/user/jenkins/test-data/dec63fa1-57fb-a401-e986-0477dde7e446/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-17T20:35:03,179 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-17T20:35:03,179 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-17T20:35:03,180 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-17T20:35:03,180 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 0b0b0ae8de03,39627,1731875702507
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-17T20:35:03,182 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/0b0b0ae8de03:0, corePoolSize=5, maxPoolSize=5
2024-11-17T20:35:03,182 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/0b0b0ae8de03:0, corePoolSize=5, maxPoolSize=5
2024-11-17T20:35:03,182 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/0b0b0ae8de03:0, corePoolSize=5, maxPoolSize=5
2024-11-17T20:35:03,182 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/0b0b0ae8de03:0, corePoolSize=5, maxPoolSize=5
2024-11-17T20:35:03,182 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/0b0b0ae8de03:0, corePoolSize=10, maxPoolSize=10
2024-11-17T20:35:03,182 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:35:03,182 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/0b0b0ae8de03:0, corePoolSize=2, maxPoolSize=2
2024-11-17T20:35:03,182 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:35:03,184 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731875733183
2024-11-17T20:35:03,184 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-17T20:35:03,184 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-17T20:35:03,184 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-17T20:35:03,184 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-17T20:35:03,184 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-17T20:35:03,184 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-17T20:35:03,184 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-17T20:35:03,185 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-17T20:35:03,185 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-17T20:35:03,185 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-17T20:35:03,185 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-17T20:35:03,185 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-17T20:35:03,185 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-17T20:35:03,185 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-17T20:35:03,185 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/0b0b0ae8de03:0:becomeActiveMaster-HFileCleaner.large.0-1731875703185,5,FailOnTimeoutGroup]
2024-11-17T20:35:03,185 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/0b0b0ae8de03:0:becomeActiveMaster-HFileCleaner.small.0-1731875703185,5,FailOnTimeoutGroup]
2024-11-17T20:35:03,186 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-17T20:35:03,186 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-17T20:35:03,186 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-17T20:35:03,186 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-17T20:35:03,186 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:35:03,186 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-17T20:35:03,193 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741831_1007 (size=1321)
2024-11-17T20:35:03,193 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41377 is added to blk_1073741831_1007 (size=1321)
2024-11-17T20:35:03,195 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:36611/user/jenkins/test-data/dec63fa1-57fb-a401-e986-0477dde7e446/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-17T20:35:03,195 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:36611/user/jenkins/test-data/dec63fa1-57fb-a401-e986-0477dde7e446
2024-11-17T20:35:03,204 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741832_1008 (size=32)
2024-11-17T20:35:03,204 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41377 is added to blk_1073741832_1008 (size=32)
2024-11-17T20:35:03,206 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T20:35:03,208 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-17T20:35:03,210 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-17T20:35:03,210 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:35:03,211 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T20:35:03,211 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-17T20:35:03,213 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-17T20:35:03,213 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:35:03,213 INFO  [RS:0;0b0b0ae8de03:45013 {}] regionserver.HRegionServer(746): ClusterId : 19318c1e-d59e-47a4-8776-f4c4db6dc4ae
2024-11-17T20:35:03,214 DEBUG [RS:0;0b0b0ae8de03:45013 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-17T20:35:03,214 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T20:35:03,214 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-17T20:35:03,216 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-17T20:35:03,216 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:35:03,217 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T20:35:03,217 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-17T20:35:03,219 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-17T20:35:03,220 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:35:03,220 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T20:35:03,221 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-17T20:35:03,222 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36611/user/jenkins/test-data/dec63fa1-57fb-a401-e986-0477dde7e446/data/hbase/meta/1588230740
2024-11-17T20:35:03,222 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36611/user/jenkins/test-data/dec63fa1-57fb-a401-e986-0477dde7e446/data/hbase/meta/1588230740
2024-11-17T20:35:03,224 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-17T20:35:03,224 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-17T20:35:03,225 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-17T20:35:03,226 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-17T20:35:03,227 DEBUG [RS:0;0b0b0ae8de03:45013 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-17T20:35:03,227 DEBUG [RS:0;0b0b0ae8de03:45013 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-17T20:35:03,229 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36611/user/jenkins/test-data/dec63fa1-57fb-a401-e986-0477dde7e446/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-17T20:35:03,230 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=822169, jitterRate=0.04544270038604736}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-17T20:35:03,231 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1731875703206Initializing all the Stores at 1731875703207 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731875703207Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731875703208 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731875703208Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731875703208Cleaning up temporary data from old regions at 1731875703224 (+16 ms)Region opened successfully at 1731875703231 (+7 ms)
2024-11-17T20:35:03,231 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-17T20:35:03,231 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-17T20:35:03,231 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-17T20:35:03,231 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-17T20:35:03,231 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-17T20:35:03,232 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-17T20:35:03,232 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731875703231Disabling compacts and flushes for region at 1731875703231Disabling writes for close at 1731875703231Writing region close event to WAL at 1731875703232 (+1 ms)Closed at 1731875703232
2024-11-17T20:35:03,234 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-17T20:35:03,234 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-17T20:35:03,234 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-17T20:35:03,236 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-17T20:35:03,238 DEBUG [RS:0;0b0b0ae8de03:45013 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-17T20:35:03,238 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-17T20:35:03,239 DEBUG [RS:0;0b0b0ae8de03:45013 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a4e1a13, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0b0b0ae8de03/172.17.0.2:0
2024-11-17T20:35:03,257 DEBUG [RS:0;0b0b0ae8de03:45013 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;0b0b0ae8de03:45013
2024-11-17T20:35:03,257 INFO  [RS:0;0b0b0ae8de03:45013 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-17T20:35:03,257 INFO  [RS:0;0b0b0ae8de03:45013 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-17T20:35:03,257 DEBUG [RS:0;0b0b0ae8de03:45013 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-17T20:35:03,258 INFO  [RS:0;0b0b0ae8de03:45013 {}] regionserver.HRegionServer(2659): reportForDuty to master=0b0b0ae8de03,39627,1731875702507 with port=45013, startcode=1731875702678
2024-11-17T20:35:03,259 DEBUG [RS:0;0b0b0ae8de03:45013 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-17T20:35:03,262 INFO  [HMaster-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54775, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService
2024-11-17T20:35:03,263 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39627 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 0b0b0ae8de03,45013,1731875702678
2024-11-17T20:35:03,264 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39627 {}] master.ServerManager(517): Registering regionserver=0b0b0ae8de03,45013,1731875702678
2024-11-17T20:35:03,269 DEBUG [RS:0;0b0b0ae8de03:45013 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:36611/user/jenkins/test-data/dec63fa1-57fb-a401-e986-0477dde7e446
2024-11-17T20:35:03,269 DEBUG [RS:0;0b0b0ae8de03:45013 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:36611
2024-11-17T20:35:03,269 DEBUG [RS:0;0b0b0ae8de03:45013 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-17T20:35:03,279 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39627-0x1014a8202800000, quorum=127.0.0.1:52057, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-17T20:35:03,280 DEBUG [RS:0;0b0b0ae8de03:45013 {}] zookeeper.ZKUtil(111): regionserver:45013-0x1014a8202800001, quorum=127.0.0.1:52057, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/0b0b0ae8de03,45013,1731875702678
2024-11-17T20:35:03,280 WARN  [RS:0;0b0b0ae8de03:45013 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-17T20:35:03,280 INFO  [RS:0;0b0b0ae8de03:45013 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-17T20:35:03,280 DEBUG [RS:0;0b0b0ae8de03:45013 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:36611/user/jenkins/test-data/dec63fa1-57fb-a401-e986-0477dde7e446/WALs/0b0b0ae8de03,45013,1731875702678
2024-11-17T20:35:03,281 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [0b0b0ae8de03,45013,1731875702678]
2024-11-17T20:35:03,292 INFO  [RS:0;0b0b0ae8de03:45013 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-17T20:35:03,304 INFO  [RS:0;0b0b0ae8de03:45013 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-17T20:35:03,305 INFO  [RS:0;0b0b0ae8de03:45013 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-17T20:35:03,305 INFO  [RS:0;0b0b0ae8de03:45013 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-17T20:35:03,305 INFO  [RS:0;0b0b0ae8de03:45013 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-17T20:35:03,307 INFO  [RS:0;0b0b0ae8de03:45013 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-17T20:35:03,307 INFO  [RS:0;0b0b0ae8de03:45013 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-17T20:35:03,307 DEBUG [RS:0;0b0b0ae8de03:45013 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:35:03,307 DEBUG [RS:0;0b0b0ae8de03:45013 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:35:03,307 DEBUG [RS:0;0b0b0ae8de03:45013 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:35:03,308 DEBUG [RS:0;0b0b0ae8de03:45013 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:35:03,308 DEBUG [RS:0;0b0b0ae8de03:45013 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:35:03,308 DEBUG [RS:0;0b0b0ae8de03:45013 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/0b0b0ae8de03:0, corePoolSize=2, maxPoolSize=2
2024-11-17T20:35:03,308 DEBUG [RS:0;0b0b0ae8de03:45013 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:35:03,308 DEBUG [RS:0;0b0b0ae8de03:45013 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:35:03,308 DEBUG [RS:0;0b0b0ae8de03:45013 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:35:03,308 DEBUG [RS:0;0b0b0ae8de03:45013 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:35:03,308 DEBUG [RS:0;0b0b0ae8de03:45013 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:35:03,308 DEBUG [RS:0;0b0b0ae8de03:45013 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:35:03,308 DEBUG [RS:0;0b0b0ae8de03:45013 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/0b0b0ae8de03:0, corePoolSize=3, maxPoolSize=3
2024-11-17T20:35:03,308 DEBUG [RS:0;0b0b0ae8de03:45013 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0, corePoolSize=3, maxPoolSize=3
2024-11-17T20:35:03,316 INFO  [RS:0;0b0b0ae8de03:45013 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-17T20:35:03,316 INFO  [RS:0;0b0b0ae8de03:45013 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-17T20:35:03,316 INFO  [RS:0;0b0b0ae8de03:45013 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-17T20:35:03,316 INFO  [RS:0;0b0b0ae8de03:45013 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-17T20:35:03,316 INFO  [RS:0;0b0b0ae8de03:45013 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-17T20:35:03,316 INFO  [RS:0;0b0b0ae8de03:45013 {}] hbase.ChoreService(168): Chore ScheduledChore name=0b0b0ae8de03,45013,1731875702678-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-17T20:35:03,338 INFO  [RS:0;0b0b0ae8de03:45013 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-17T20:35:03,338 INFO  [RS:0;0b0b0ae8de03:45013 {}] hbase.ChoreService(168): Chore ScheduledChore name=0b0b0ae8de03,45013,1731875702678-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-17T20:35:03,338 INFO  [RS:0;0b0b0ae8de03:45013 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T20:35:03,338 INFO  [RS:0;0b0b0ae8de03:45013 {}] regionserver.Replication(171): 0b0b0ae8de03,45013,1731875702678 started
2024-11-17T20:35:03,357 INFO  [RS:0;0b0b0ae8de03:45013 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T20:35:03,357 INFO  [RS:0;0b0b0ae8de03:45013 {}] regionserver.HRegionServer(1482): Serving as 0b0b0ae8de03,45013,1731875702678, RpcServer on 0b0b0ae8de03/172.17.0.2:45013, sessionid=0x1014a8202800001
2024-11-17T20:35:03,357 DEBUG [RS:0;0b0b0ae8de03:45013 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-17T20:35:03,357 DEBUG [RS:0;0b0b0ae8de03:45013 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 0b0b0ae8de03,45013,1731875702678
2024-11-17T20:35:03,357 DEBUG [RS:0;0b0b0ae8de03:45013 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0b0b0ae8de03,45013,1731875702678'
2024-11-17T20:35:03,357 DEBUG [RS:0;0b0b0ae8de03:45013 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-17T20:35:03,358 DEBUG [RS:0;0b0b0ae8de03:45013 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-17T20:35:03,359 DEBUG [RS:0;0b0b0ae8de03:45013 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-17T20:35:03,359 DEBUG [RS:0;0b0b0ae8de03:45013 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-17T20:35:03,359 DEBUG [RS:0;0b0b0ae8de03:45013 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 0b0b0ae8de03,45013,1731875702678
2024-11-17T20:35:03,359 DEBUG [RS:0;0b0b0ae8de03:45013 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0b0b0ae8de03,45013,1731875702678'
2024-11-17T20:35:03,359 DEBUG [RS:0;0b0b0ae8de03:45013 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-17T20:35:03,360 DEBUG [RS:0;0b0b0ae8de03:45013 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-17T20:35:03,362 DEBUG [RS:0;0b0b0ae8de03:45013 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-17T20:35:03,362 INFO  [RS:0;0b0b0ae8de03:45013 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-17T20:35:03,362 INFO  [RS:0;0b0b0ae8de03:45013 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-17T20:35:03,388 WARN  [0b0b0ae8de03:39627 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions.
2024-11-17T20:35:03,465 INFO  [RS:0;0b0b0ae8de03:45013 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0b0b0ae8de03%2C45013%2C1731875702678, suffix=, logDir=hdfs://localhost:36611/user/jenkins/test-data/dec63fa1-57fb-a401-e986-0477dde7e446/WALs/0b0b0ae8de03,45013,1731875702678, archiveDir=hdfs://localhost:36611/user/jenkins/test-data/dec63fa1-57fb-a401-e986-0477dde7e446/oldWALs, maxLogs=32
2024-11-17T20:35:03,468 INFO  [RS:0;0b0b0ae8de03:45013 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0b0b0ae8de03%2C45013%2C1731875702678.1731875703467
2024-11-17T20:35:03,481 INFO  [RS:0;0b0b0ae8de03:45013 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/dec63fa1-57fb-a401-e986-0477dde7e446/WALs/0b0b0ae8de03,45013,1731875702678/0b0b0ae8de03%2C45013%2C1731875702678.1731875703467
2024-11-17T20:35:03,485 DEBUG [RS:0;0b0b0ae8de03:45013 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42565:42565),(127.0.0.1/127.0.0.1:38843:38843)]
2024-11-17T20:35:03,639 DEBUG [0b0b0ae8de03:39627 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-17T20:35:03,639 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=0b0b0ae8de03,45013,1731875702678
2024-11-17T20:35:03,641 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0b0b0ae8de03,45013,1731875702678, state=OPENING
2024-11-17T20:35:03,689 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-17T20:35:03,700 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39627-0x1014a8202800000, quorum=127.0.0.1:52057, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:35:03,700 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45013-0x1014a8202800001, quorum=127.0.0.1:52057, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:35:03,700 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-17T20:35:03,700 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-17T20:35:03,700 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-17T20:35:03,701 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=0b0b0ae8de03,45013,1731875702678}]
2024-11-17T20:35:03,804 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-17T20:35:03,804 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-17T20:35:03,805 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling
2024-11-17T20:35:03,855 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-17T20:35:03,857 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56387, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-17T20:35:03,864 INFO  [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-17T20:35:03,864 INFO  [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-17T20:35:03,867 INFO  [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0b0b0ae8de03%2C45013%2C1731875702678.meta, suffix=.meta, logDir=hdfs://localhost:36611/user/jenkins/test-data/dec63fa1-57fb-a401-e986-0477dde7e446/WALs/0b0b0ae8de03,45013,1731875702678, archiveDir=hdfs://localhost:36611/user/jenkins/test-data/dec63fa1-57fb-a401-e986-0477dde7e446/oldWALs, maxLogs=32
2024-11-17T20:35:03,869 INFO  [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 0b0b0ae8de03%2C45013%2C1731875702678.meta.1731875703869.meta
2024-11-17T20:35:03,882 INFO  [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/dec63fa1-57fb-a401-e986-0477dde7e446/WALs/0b0b0ae8de03,45013,1731875702678/0b0b0ae8de03%2C45013%2C1731875702678.meta.1731875703869.meta
2024-11-17T20:35:03,888 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42565:42565),(127.0.0.1/127.0.0.1:38843:38843)]
2024-11-17T20:35:03,889 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-17T20:35:03,890 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-17T20:35:03,890 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-17T20:35:03,890 INFO  [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-17T20:35:03,890 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-17T20:35:03,890 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T20:35:03,890 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-17T20:35:03,890 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-17T20:35:03,892 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-17T20:35:03,894 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-17T20:35:03,894 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:35:03,894 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T20:35:03,894 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-17T20:35:03,896 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-17T20:35:03,896 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:35:03,897 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T20:35:03,897 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-17T20:35:03,898 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-17T20:35:03,898 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:35:03,899 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T20:35:03,899 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-17T20:35:03,900 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-17T20:35:03,901 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:35:03,901 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T20:35:03,901 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-17T20:35:03,903 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36611/user/jenkins/test-data/dec63fa1-57fb-a401-e986-0477dde7e446/data/hbase/meta/1588230740
2024-11-17T20:35:03,905 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36611/user/jenkins/test-data/dec63fa1-57fb-a401-e986-0477dde7e446/data/hbase/meta/1588230740
2024-11-17T20:35:03,907 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-17T20:35:03,907 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-17T20:35:03,908 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-17T20:35:03,910 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-17T20:35:03,912 INFO  [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=735824, jitterRate=-0.06435248255729675}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-17T20:35:03,912 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-17T20:35:03,914 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1731875703890Writing region info on filesystem at 1731875703890Initializing all the Stores at 1731875703892 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731875703892Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731875703892Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731875703892Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731875703892Cleaning up temporary data from old regions at 1731875703907 (+15 ms)Running coprocessor post-open hooks at 1731875703912 (+5 ms)Region opened successfully at 1731875703913 (+1 ms)
2024-11-17T20:35:03,915 INFO  [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731875703854
2024-11-17T20:35:03,919 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-17T20:35:03,919 INFO  [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-17T20:35:03,920 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=0b0b0ae8de03,45013,1731875702678
2024-11-17T20:35:03,922 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0b0b0ae8de03,45013,1731875702678, state=OPEN
2024-11-17T20:35:03,964 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39627-0x1014a8202800000, quorum=127.0.0.1:52057, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-17T20:35:03,964 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45013-0x1014a8202800001, quorum=127.0.0.1:52057, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-17T20:35:03,964 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=0b0b0ae8de03,45013,1731875702678
2024-11-17T20:35:03,964 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-17T20:35:03,964 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-17T20:35:03,969 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-17T20:35:03,969 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=0b0b0ae8de03,45013,1731875702678 in 264 msec
2024-11-17T20:35:03,972 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-17T20:35:03,972 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 735 msec
2024-11-17T20:35:03,973 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-17T20:35:03,973 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-17T20:35:03,975 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-17T20:35:03,975 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,45013,1731875702678, seqNum=-1]
2024-11-17T20:35:03,976 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-17T20:35:03,977 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40047, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-17T20:35:03,986 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 806 msec
2024-11-17T20:35:03,986 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731875703986, completionTime=-1
2024-11-17T20:35:03,986 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-17T20:35:03,986 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster...
2024-11-17T20:35:03,990 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1
2024-11-17T20:35:03,990 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731875763990
2024-11-17T20:35:03,990 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731875823990
2024-11-17T20:35:03,990 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 3 msec
2024-11-17T20:35:03,990 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0b0b0ae8de03,39627,1731875702507-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-17T20:35:03,991 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0b0b0ae8de03,39627,1731875702507-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T20:35:03,991 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0b0b0ae8de03,39627,1731875702507-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T20:35:03,991 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-0b0b0ae8de03:39627, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T20:35:03,991 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-17T20:35:03,992 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-17T20:35:03,999 DEBUG [master/0b0b0ae8de03:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-17T20:35:04,002 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.243sec
2024-11-17T20:35:04,002 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-17T20:35:04,002 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-17T20:35:04,002 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-17T20:35:04,002 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-17T20:35:04,002 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-17T20:35:04,002 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0b0b0ae8de03,39627,1731875702507-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-17T20:35:04,002 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0b0b0ae8de03,39627,1731875702507-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-17T20:35:04,005 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-17T20:35:04,005 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-17T20:35:04,006 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0b0b0ae8de03,39627,1731875702507-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T20:35:04,014 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e299140, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-17T20:35:04,014 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b0b0ae8de03,39627,-1 for getting cluster id
2024-11-17T20:35:04,015 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-17T20:35:04,025 DEBUG [HMaster-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '19318c1e-d59e-47a4-8776-f4c4db6dc4ae'
2024-11-17T20:35:04,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-17T20:35:04,026 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "19318c1e-d59e-47a4-8776-f4c4db6dc4ae"

2024-11-17T20:35:04,026 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66386db6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-17T20:35:04,026 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b0b0ae8de03,39627,-1]
2024-11-17T20:35:04,026 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-17T20:35:04,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T20:35:04,030 INFO  [HMaster-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39398, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-17T20:35:04,031 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73a78491, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-17T20:35:04,032 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-17T20:35:04,034 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,45013,1731875702678, seqNum=-1]
2024-11-17T20:35:04,034 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-17T20:35:04,037 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55122, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-17T20:35:04,040 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=0b0b0ae8de03,39627,1731875702507
2024-11-17T20:35:04,041 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T20:35:04,044 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-17T20:35:04,045 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-17T20:35:04,045 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-17T20:35:04,045 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at org.apache.hadoop.hbase.regionserver.wal.TestLogRolling.testLogRollOnDatanodeDeath(TestLogRolling.java:201)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-17T20:35:04,045 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T20:35:04,046 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T20:35:04,046 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-17T20:35:04,046 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-17T20:35:04,046 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=904512183, stopped=false
2024-11-17T20:35:04,046 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=0b0b0ae8de03,39627,1731875702507
2024-11-17T20:35:04,068 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39627-0x1014a8202800000, quorum=127.0.0.1:52057, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-17T20:35:04,068 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45013-0x1014a8202800001, quorum=127.0.0.1:52057, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-17T20:35:04,068 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45013-0x1014a8202800001, quorum=127.0.0.1:52057, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:35:04,069 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39627-0x1014a8202800000, quorum=127.0.0.1:52057, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:35:04,069 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-17T20:35:04,069 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:39627-0x1014a8202800000, quorum=127.0.0.1:52057, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-17T20:35:04,070 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45013-0x1014a8202800001, quorum=127.0.0.1:52057, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-17T20:35:04,070 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-17T20:35:04,070 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at org.apache.hadoop.hbase.regionserver.wal.TestLogRolling.testLogRollOnDatanodeDeath(TestLogRolling.java:201)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-17T20:35:04,070 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T20:35:04,071 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '0b0b0ae8de03,45013,1731875702678' *****
2024-11-17T20:35:04,071 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-17T20:35:04,071 INFO  [RS:0;0b0b0ae8de03:45013 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-17T20:35:04,071 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-17T20:35:04,071 INFO  [RS:0;0b0b0ae8de03:45013 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-17T20:35:04,071 INFO  [RS:0;0b0b0ae8de03:45013 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-17T20:35:04,071 INFO  [RS:0;0b0b0ae8de03:45013 {}] regionserver.HRegionServer(959): stopping server 0b0b0ae8de03,45013,1731875702678
2024-11-17T20:35:04,071 INFO  [RS:0;0b0b0ae8de03:45013 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-17T20:35:04,072 INFO  [RS:0;0b0b0ae8de03:45013 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;0b0b0ae8de03:45013.
2024-11-17T20:35:04,072 DEBUG [RS:0;0b0b0ae8de03:45013 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-17T20:35:04,072 DEBUG [RS:0;0b0b0ae8de03:45013 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T20:35:04,072 INFO  [RS:0;0b0b0ae8de03:45013 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-17T20:35:04,072 INFO  [RS:0;0b0b0ae8de03:45013 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-17T20:35:04,072 INFO  [RS:0;0b0b0ae8de03:45013 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-17T20:35:04,072 INFO  [RS:0;0b0b0ae8de03:45013 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-17T20:35:04,073 INFO  [RS:0;0b0b0ae8de03:45013 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close
2024-11-17T20:35:04,073 DEBUG [RS:0;0b0b0ae8de03:45013 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740}
2024-11-17T20:35:04,073 DEBUG [RS:0;0b0b0ae8de03:45013 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-17T20:35:04,073 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-17T20:35:04,073 INFO  [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-17T20:35:04,073 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-17T20:35:04,073 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-17T20:35:04,073 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-17T20:35:04,073 INFO  [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=74 B heapSize=1.22 KB
2024-11-17T20:35:04,096 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36611/user/jenkins/test-data/dec63fa1-57fb-a401-e986-0477dde7e446/data/hbase/meta/1588230740/.tmp/ns/0d1e49efdcc4475f8972b43db697facc is 43, key is default/ns:d/1731875703978/Put/seqid=0
2024-11-17T20:35:04,106 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41377 is added to blk_1073741835_1011 (size=5153)
2024-11-17T20:35:04,108 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741835_1011 (size=5153)
2024-11-17T20:35:04,109 INFO  [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:36611/user/jenkins/test-data/dec63fa1-57fb-a401-e986-0477dde7e446/data/hbase/meta/1588230740/.tmp/ns/0d1e49efdcc4475f8972b43db697facc
2024-11-17T20:35:04,121 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36611/user/jenkins/test-data/dec63fa1-57fb-a401-e986-0477dde7e446/data/hbase/meta/1588230740/.tmp/ns/0d1e49efdcc4475f8972b43db697facc as hdfs://localhost:36611/user/jenkins/test-data/dec63fa1-57fb-a401-e986-0477dde7e446/data/hbase/meta/1588230740/ns/0d1e49efdcc4475f8972b43db697facc
2024-11-17T20:35:04,134 INFO  [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36611/user/jenkins/test-data/dec63fa1-57fb-a401-e986-0477dde7e446/data/hbase/meta/1588230740/ns/0d1e49efdcc4475f8972b43db697facc, entries=2, sequenceid=6, filesize=5.0 K
2024-11-17T20:35:04,135 INFO  [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 62ms, sequenceid=6, compaction requested=false
2024-11-17T20:35:04,136 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-17T20:35:04,143 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36611/user/jenkins/test-data/dec63fa1-57fb-a401-e986-0477dde7e446/data/hbase/meta/1588230740/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1
2024-11-17T20:35:04,143 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-17T20:35:04,144 INFO  [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-17T20:35:04,144 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731875704073Running coprocessor pre-close hooks at 1731875704073Disabling compacts and flushes for region at 1731875704073Disabling writes for close at 1731875704073Obtaining lock to block concurrent updates at 1731875704073Preparing flush snapshotting stores in 1588230740 at 1731875704073Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=74, getHeapSize=1184, getOffHeapSize=0, getCellsCount=2 at 1731875704074 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731875704075 (+1 ms)Flushing 1588230740/ns: creating writer at 1731875704075Flushing 1588230740/ns: appending metadata at 1731875704095 (+20 ms)Flushing 1588230740/ns: closing flushed file at 1731875704095Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@10a7f5ad: reopening flushed file at 1731875704120 (+25 ms)Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 62ms, sequenceid=6, compaction requested=false at 1731875704135 (+15 ms)Writing region close event to WAL at 1731875704137 (+2 ms)Running coprocessor post-close hooks at 1731875704143 (+6 ms)Closed at 1731875704144 (+1 ms)
2024-11-17T20:35:04,144 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-17T20:35:04,273 INFO  [RS:0;0b0b0ae8de03:45013 {}] regionserver.HRegionServer(976): stopping server 0b0b0ae8de03,45013,1731875702678; all regions closed.
2024-11-17T20:35:04,274 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:35:04,274 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:35:04,274 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:35:04,274 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:35:04,274 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:35:04,277 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41377 is added to blk_1073741834_1010 (size=1152)
2024-11-17T20:35:04,277 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741834_1010 (size=1152)
2024-11-17T20:35:04,281 DEBUG [RS:0;0b0b0ae8de03:45013 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/dec63fa1-57fb-a401-e986-0477dde7e446/oldWALs
2024-11-17T20:35:04,281 INFO  [RS:0;0b0b0ae8de03:45013 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 0b0b0ae8de03%2C45013%2C1731875702678.meta:.meta(num 1731875703869)
2024-11-17T20:35:04,282 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:35:04,282 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:35:04,282 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:35:04,282 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:35:04,283 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:35:04,285 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41377 is added to blk_1073741833_1009 (size=93)
2024-11-17T20:35:04,285 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741833_1009 (size=93)
2024-11-17T20:35:04,289 DEBUG [RS:0;0b0b0ae8de03:45013 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/dec63fa1-57fb-a401-e986-0477dde7e446/oldWALs
2024-11-17T20:35:04,289 INFO  [RS:0;0b0b0ae8de03:45013 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 0b0b0ae8de03%2C45013%2C1731875702678:(num 1731875703467)
2024-11-17T20:35:04,289 DEBUG [RS:0;0b0b0ae8de03:45013 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T20:35:04,289 INFO  [RS:0;0b0b0ae8de03:45013 {}] regionserver.LeaseManager(133): Closed leases
2024-11-17T20:35:04,289 INFO  [RS:0;0b0b0ae8de03:45013 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-17T20:35:04,289 INFO  [RS:0;0b0b0ae8de03:45013 {}] hbase.ChoreService(370): Chore service for: regionserver/0b0b0ae8de03:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-17T20:35:04,289 INFO  [RS:0;0b0b0ae8de03:45013 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-17T20:35:04,289 INFO  [regionserver/0b0b0ae8de03:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-17T20:35:04,290 INFO  [RS:0;0b0b0ae8de03:45013 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45013
2024-11-17T20:35:04,373 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45013-0x1014a8202800001, quorum=127.0.0.1:52057, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/0b0b0ae8de03,45013,1731875702678
2024-11-17T20:35:04,373 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39627-0x1014a8202800000, quorum=127.0.0.1:52057, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-17T20:35:04,373 INFO  [RS:0;0b0b0ae8de03:45013 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-17T20:35:04,373 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [0b0b0ae8de03,45013,1731875702678]
2024-11-17T20:35:04,516 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/0b0b0ae8de03,45013,1731875702678 already deleted, retry=false
2024-11-17T20:35:04,516 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 0b0b0ae8de03,45013,1731875702678 expired; onlineServers=0
2024-11-17T20:35:04,516 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '0b0b0ae8de03,39627,1731875702507' *****
2024-11-17T20:35:04,517 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-17T20:35:04,517 INFO  [M:0;0b0b0ae8de03:39627 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-17T20:35:04,517 INFO  [M:0;0b0b0ae8de03:39627 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-17T20:35:04,517 DEBUG [M:0;0b0b0ae8de03:39627 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-17T20:35:04,517 DEBUG [M:0;0b0b0ae8de03:39627 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-17T20:35:04,517 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-17T20:35:04,517 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster-HFileCleaner.small.0-1731875703185 {}] cleaner.HFileCleaner(306): Exit Thread[master/0b0b0ae8de03:0:becomeActiveMaster-HFileCleaner.small.0-1731875703185,5,FailOnTimeoutGroup]
2024-11-17T20:35:04,517 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster-HFileCleaner.large.0-1731875703185 {}] cleaner.HFileCleaner(306): Exit Thread[master/0b0b0ae8de03:0:becomeActiveMaster-HFileCleaner.large.0-1731875703185,5,FailOnTimeoutGroup]
2024-11-17T20:35:04,517 INFO  [M:0;0b0b0ae8de03:39627 {}] hbase.ChoreService(370): Chore service for: master/0b0b0ae8de03:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-17T20:35:04,517 INFO  [M:0;0b0b0ae8de03:39627 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-17T20:35:04,518 DEBUG [M:0;0b0b0ae8de03:39627 {}] master.HMaster(1795): Stopping service threads
2024-11-17T20:35:04,518 INFO  [M:0;0b0b0ae8de03:39627 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-17T20:35:04,518 INFO  [M:0;0b0b0ae8de03:39627 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-17T20:35:04,518 INFO  [M:0;0b0b0ae8de03:39627 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-17T20:35:04,518 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-17T20:35:04,527 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39627-0x1014a8202800000, quorum=127.0.0.1:52057, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-17T20:35:04,527 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39627-0x1014a8202800000, quorum=127.0.0.1:52057, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:35:04,527 DEBUG [M:0;0b0b0ae8de03:39627 {}] zookeeper.ZKUtil(347): master:39627-0x1014a8202800000, quorum=127.0.0.1:52057, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-17T20:35:04,527 WARN  [M:0;0b0b0ae8de03:39627 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-17T20:35:04,528 INFO  [M:0;0b0b0ae8de03:39627 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:36611/user/jenkins/test-data/dec63fa1-57fb-a401-e986-0477dde7e446/.lastflushedseqids
2024-11-17T20:35:04,535 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41377 is added to blk_1073741836_1012 (size=99)
2024-11-17T20:35:04,535 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741836_1012 (size=99)
2024-11-17T20:35:04,536 INFO  [M:0;0b0b0ae8de03:39627 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-17T20:35:04,536 INFO  [M:0;0b0b0ae8de03:39627 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-17T20:35:04,536 DEBUG [M:0;0b0b0ae8de03:39627 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-17T20:35:04,536 INFO  [M:0;0b0b0ae8de03:39627 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T20:35:04,536 DEBUG [M:0;0b0b0ae8de03:39627 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T20:35:04,536 DEBUG [M:0;0b0b0ae8de03:39627 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-17T20:35:04,536 DEBUG [M:0;0b0b0ae8de03:39627 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T20:35:04,536 INFO  [M:0;0b0b0ae8de03:39627 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=7.67 KB heapSize=11.34 KB
2024-11-17T20:35:04,559 DEBUG [M:0;0b0b0ae8de03:39627 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36611/user/jenkins/test-data/dec63fa1-57fb-a401-e986-0477dde7e446/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/755cd77bd4774ce0b13933446201e7ed is 82, key is hbase:meta,,1/info:regioninfo/1731875703920/Put/seqid=0
2024-11-17T20:35:04,566 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741837_1013 (size=5672)
2024-11-17T20:35:04,566 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41377 is added to blk_1073741837_1013 (size=5672)
2024-11-17T20:35:04,567 INFO  [M:0;0b0b0ae8de03:39627 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:36611/user/jenkins/test-data/dec63fa1-57fb-a401-e986-0477dde7e446/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/755cd77bd4774ce0b13933446201e7ed
2024-11-17T20:35:04,588 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:35:04,590 INFO  [RS:0;0b0b0ae8de03:45013 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-17T20:35:04,590 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45013-0x1014a8202800001, quorum=127.0.0.1:52057, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-17T20:35:04,590 INFO  [RS:0;0b0b0ae8de03:45013 {}] regionserver.HRegionServer(1031): Exiting; stopping=0b0b0ae8de03,45013,1731875702678; zookeeper connection closed.
2024-11-17T20:35:04,590 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45013-0x1014a8202800001, quorum=127.0.0.1:52057, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-17T20:35:04,591 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@47760ea6 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@47760ea6
2024-11-17T20:35:04,592 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-17T20:35:04,593 DEBUG [M:0;0b0b0ae8de03:39627 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36611/user/jenkins/test-data/dec63fa1-57fb-a401-e986-0477dde7e446/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/586be59793f14a21915cf127be7ae20c is 240, key is \x00\x00\x00\x00\x00\x00\x00\x01/proc:d/1731875703985/Put/seqid=0
2024-11-17T20:35:04,594 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:35:04,602 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41377 is added to blk_1073741838_1014 (size=5275)
2024-11-17T20:35:04,605 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741838_1014 (size=5275)
2024-11-17T20:35:04,606 INFO  [M:0;0b0b0ae8de03:39627 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.06 KB at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:36611/user/jenkins/test-data/dec63fa1-57fb-a401-e986-0477dde7e446/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/586be59793f14a21915cf127be7ae20c
2024-11-17T20:35:04,629 DEBUG [M:0;0b0b0ae8de03:39627 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36611/user/jenkins/test-data/dec63fa1-57fb-a401-e986-0477dde7e446/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/36687a970f17405f8347a28284de8b7a is 69, key is 0b0b0ae8de03,45013,1731875702678/rs:state/1731875703264/Put/seqid=0
2024-11-17T20:35:04,635 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41377 is added to blk_1073741839_1015 (size=5156)
2024-11-17T20:35:04,635 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741839_1015 (size=5156)
2024-11-17T20:35:04,636 INFO  [M:0;0b0b0ae8de03:39627 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:36611/user/jenkins/test-data/dec63fa1-57fb-a401-e986-0477dde7e446/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/36687a970f17405f8347a28284de8b7a
2024-11-17T20:35:04,663 DEBUG [M:0;0b0b0ae8de03:39627 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36611/user/jenkins/test-data/dec63fa1-57fb-a401-e986-0477dde7e446/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/4579922695b44abfa489809811fd1299 is 52, key is load_balancer_on/state:d/1731875704043/Put/seqid=0
2024-11-17T20:35:04,669 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41377 is added to blk_1073741840_1016 (size=5056)
2024-11-17T20:35:04,669 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741840_1016 (size=5056)
2024-11-17T20:35:04,669 INFO  [M:0;0b0b0ae8de03:39627 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:36611/user/jenkins/test-data/dec63fa1-57fb-a401-e986-0477dde7e446/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/4579922695b44abfa489809811fd1299
2024-11-17T20:35:04,678 DEBUG [M:0;0b0b0ae8de03:39627 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36611/user/jenkins/test-data/dec63fa1-57fb-a401-e986-0477dde7e446/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/755cd77bd4774ce0b13933446201e7ed as hdfs://localhost:36611/user/jenkins/test-data/dec63fa1-57fb-a401-e986-0477dde7e446/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/755cd77bd4774ce0b13933446201e7ed
2024-11-17T20:35:04,685 INFO  [M:0;0b0b0ae8de03:39627 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36611/user/jenkins/test-data/dec63fa1-57fb-a401-e986-0477dde7e446/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/755cd77bd4774ce0b13933446201e7ed, entries=8, sequenceid=29, filesize=5.5 K
2024-11-17T20:35:04,687 DEBUG [M:0;0b0b0ae8de03:39627 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36611/user/jenkins/test-data/dec63fa1-57fb-a401-e986-0477dde7e446/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/586be59793f14a21915cf127be7ae20c as hdfs://localhost:36611/user/jenkins/test-data/dec63fa1-57fb-a401-e986-0477dde7e446/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/586be59793f14a21915cf127be7ae20c
2024-11-17T20:35:04,695 INFO  [M:0;0b0b0ae8de03:39627 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36611/user/jenkins/test-data/dec63fa1-57fb-a401-e986-0477dde7e446/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/586be59793f14a21915cf127be7ae20c, entries=3, sequenceid=29, filesize=5.2 K
2024-11-17T20:35:04,696 DEBUG [M:0;0b0b0ae8de03:39627 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36611/user/jenkins/test-data/dec63fa1-57fb-a401-e986-0477dde7e446/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/36687a970f17405f8347a28284de8b7a as hdfs://localhost:36611/user/jenkins/test-data/dec63fa1-57fb-a401-e986-0477dde7e446/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/36687a970f17405f8347a28284de8b7a
2024-11-17T20:35:04,703 INFO  [M:0;0b0b0ae8de03:39627 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36611/user/jenkins/test-data/dec63fa1-57fb-a401-e986-0477dde7e446/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/36687a970f17405f8347a28284de8b7a, entries=1, sequenceid=29, filesize=5.0 K
2024-11-17T20:35:04,705 DEBUG [M:0;0b0b0ae8de03:39627 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36611/user/jenkins/test-data/dec63fa1-57fb-a401-e986-0477dde7e446/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/4579922695b44abfa489809811fd1299 as hdfs://localhost:36611/user/jenkins/test-data/dec63fa1-57fb-a401-e986-0477dde7e446/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/4579922695b44abfa489809811fd1299
2024-11-17T20:35:04,712 INFO  [M:0;0b0b0ae8de03:39627 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36611/user/jenkins/test-data/dec63fa1-57fb-a401-e986-0477dde7e446/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/4579922695b44abfa489809811fd1299, entries=1, sequenceid=29, filesize=4.9 K
2024-11-17T20:35:04,713 INFO  [M:0;0b0b0ae8de03:39627 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 177ms, sequenceid=29, compaction requested=false
2024-11-17T20:35:04,714 INFO  [M:0;0b0b0ae8de03:39627 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T20:35:04,715 DEBUG [M:0;0b0b0ae8de03:39627 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731875704536Disabling compacts and flushes for region at 1731875704536Disabling writes for close at 1731875704536Obtaining lock to block concurrent updates at 1731875704536Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731875704536Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=7850, getHeapSize=11544, getOffHeapSize=0, getCellsCount=36 at 1731875704537 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731875704537Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731875704538 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731875704558 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731875704559 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731875704574 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731875704592 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731875704593 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731875704613 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731875704628 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731875704628Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731875704643 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731875704662 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731875704662Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@30ec878f: reopening flushed file at 1731875704676 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@321d16a5: reopening flushed file at 1731875704685 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4e075b71: reopening flushed file at 1731875704695 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@15af4fd5: reopening flushed file at 1731875704704 (+9 ms)Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 177ms, sequenceid=29, compaction requested=false at 1731875704713 (+9 ms)Writing region close event to WAL at 1731875704714 (+1 ms)Closed at 1731875704714
2024-11-17T20:35:04,715 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:35:04,715 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:35:04,715 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:35:04,715 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:35:04,715 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:35:04,718 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41377 is added to blk_1073741830_1006 (size=10311)
2024-11-17T20:35:04,718 INFO  [M:0;0b0b0ae8de03:39627 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-17T20:35:04,718 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-17T20:35:04,718 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741830_1006 (size=10311)
2024-11-17T20:35:04,719 INFO  [M:0;0b0b0ae8de03:39627 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39627
2024-11-17T20:35:04,719 INFO  [M:0;0b0b0ae8de03:39627 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-17T20:35:04,831 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39627-0x1014a8202800000, quorum=127.0.0.1:52057, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-17T20:35:04,831 INFO  [M:0;0b0b0ae8de03:39627 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-17T20:35:04,831 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39627-0x1014a8202800000, quorum=127.0.0.1:52057, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-17T20:35:04,836 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@210f4ee9{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T20:35:04,837 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1dedf320{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-17T20:35:04,837 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-17T20:35:04,837 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2220be00{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-17T20:35:04,837 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@17d00685{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2e45e89-d349-682c-1566-7129f4ac0660/hadoop.log.dir/,STOPPED}
2024-11-17T20:35:04,839 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-17T20:35:04,839 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-17T20:35:04,844 WARN  [BP-1418661057-172.17.0.2-1731875700237 heartbeating to localhost/127.0.0.1:36611 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-17T20:35:04,844 WARN  [BP-1418661057-172.17.0.2-1731875700237 heartbeating to localhost/127.0.0.1:36611 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1418661057-172.17.0.2-1731875700237 (Datanode Uuid 38a3733d-980e-474e-a4a6-57da3818e57a) service to localhost/127.0.0.1:36611
2024-11-17T20:35:04,845 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2e45e89-d349-682c-1566-7129f4ac0660/cluster_0dce1bfb-1fca-ebc8-fb54-79ce3a8be4db/data/data3/current/BP-1418661057-172.17.0.2-1731875700237 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T20:35:04,845 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2e45e89-d349-682c-1566-7129f4ac0660/cluster_0dce1bfb-1fca-ebc8-fb54-79ce3a8be4db/data/data4/current/BP-1418661057-172.17.0.2-1731875700237 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T20:35:04,846 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-17T20:35:04,848 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@426dc4fa{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T20:35:04,848 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@417f4bff{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-17T20:35:04,848 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-17T20:35:04,848 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3b01355c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-17T20:35:04,848 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@21e00560{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2e45e89-d349-682c-1566-7129f4ac0660/hadoop.log.dir/,STOPPED}
2024-11-17T20:35:04,853 WARN  [BP-1418661057-172.17.0.2-1731875700237 heartbeating to localhost/127.0.0.1:36611 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-17T20:35:04,853 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-17T20:35:04,853 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-17T20:35:04,853 WARN  [BP-1418661057-172.17.0.2-1731875700237 heartbeating to localhost/127.0.0.1:36611 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1418661057-172.17.0.2-1731875700237 (Datanode Uuid d7d06dc4-9366-414f-a7e6-256ba7e4b4fc) service to localhost/127.0.0.1:36611
2024-11-17T20:35:04,854 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2e45e89-d349-682c-1566-7129f4ac0660/cluster_0dce1bfb-1fca-ebc8-fb54-79ce3a8be4db/data/data1/current/BP-1418661057-172.17.0.2-1731875700237 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T20:35:04,854 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2e45e89-d349-682c-1566-7129f4ac0660/cluster_0dce1bfb-1fca-ebc8-fb54-79ce3a8be4db/data/data2/current/BP-1418661057-172.17.0.2-1731875700237 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T20:35:04,854 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-17T20:35:04,860 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@81550dd{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-17T20:35:04,861 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2d71062d{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-17T20:35:04,861 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-17T20:35:04,861 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@542ee468{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-17T20:35:04,861 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2152d149{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2e45e89-d349-682c-1566-7129f4ac0660/hadoop.log.dir/,STOPPED}
2024-11-17T20:35:04,867 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-17T20:35:04,889 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-17T20:35:04,889 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-17T20:35:04,889 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2e45e89-d349-682c-1566-7129f4ac0660/hadoop.log.dir so I do NOT create it in target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25
2024-11-17T20:35:04,889 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2e45e89-d349-682c-1566-7129f4ac0660/hadoop.tmp.dir so I do NOT create it in target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25
2024-11-17T20:35:04,889 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/cluster_b9ca40e8-4224-d8ab-7466-d1512b745e25, deleteOnExit=true
2024-11-17T20:35:04,889 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-17T20:35:04,889 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/test.cache.data in system properties and HBase conf
2024-11-17T20:35:04,889 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/hadoop.tmp.dir in system properties and HBase conf
2024-11-17T20:35:04,890 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/hadoop.log.dir in system properties and HBase conf
2024-11-17T20:35:04,890 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-17T20:35:04,890 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-17T20:35:04,890 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-17T20:35:04,890 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-17T20:35:04,890 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-17T20:35:04,891 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-17T20:35:04,891 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-17T20:35:04,891 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-17T20:35:04,891 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-17T20:35:04,891 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-17T20:35:04,891 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-17T20:35:04,891 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-17T20:35:04,891 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-17T20:35:04,892 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/nfs.dump.dir in system properties and HBase conf
2024-11-17T20:35:04,892 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/java.io.tmpdir in system properties and HBase conf
2024-11-17T20:35:04,892 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-17T20:35:04,892 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-17T20:35:04,892 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-17T20:35:04,906 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-17T20:35:05,110 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-17T20:35:05,112 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:35:05,127 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:35:05,129 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:35:05,130 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:35:05,300 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-17T20:35:05,308 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-17T20:35:05,315 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-17T20:35:05,315 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-17T20:35:05,315 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-17T20:35:05,316 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-17T20:35:05,317 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1d790455{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/hadoop.log.dir/,AVAILABLE}
2024-11-17T20:35:05,317 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@8f6e525{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-17T20:35:05,320 INFO  [regionserver/0b0b0ae8de03:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-17T20:35:05,431 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@66182b08{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/java.io.tmpdir/jetty-localhost-39987-hadoop-hdfs-3_4_1-tests_jar-_-any-16829624661709117914/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-17T20:35:05,431 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1100af17{HTTP/1.1, (http/1.1)}{localhost:39987}
2024-11-17T20:35:05,432 INFO  [Time-limited test {}] server.Server(415): Started @108718ms
2024-11-17T20:35:05,452 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-17T20:35:05,751 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-17T20:35:05,756 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-17T20:35:05,761 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-17T20:35:05,761 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-17T20:35:05,761 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-17T20:35:05,765 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4577488a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/hadoop.log.dir/,AVAILABLE}
2024-11-17T20:35:05,765 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@805be33{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-17T20:35:05,892 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@68605b4b{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/java.io.tmpdir/jetty-localhost-34593-hadoop-hdfs-3_4_1-tests_jar-_-any-15919880386330906847/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T20:35:05,892 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@158b5ab3{HTTP/1.1, (http/1.1)}{localhost:34593}
2024-11-17T20:35:05,893 INFO  [Time-limited test {}] server.Server(415): Started @109178ms
2024-11-17T20:35:05,895 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-17T20:35:05,955 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-17T20:35:05,959 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-17T20:35:05,960 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-17T20:35:05,961 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-17T20:35:05,961 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-17T20:35:05,961 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@461a42e8{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/hadoop.log.dir/,AVAILABLE}
2024-11-17T20:35:05,962 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@45b2e9f0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-17T20:35:06,073 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5a63af9{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/java.io.tmpdir/jetty-localhost-37615-hadoop-hdfs-3_4_1-tests_jar-_-any-18109153320051215147/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T20:35:06,074 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3efce601{HTTP/1.1, (http/1.1)}{localhost:37615}
2024-11-17T20:35:06,074 INFO  [Time-limited test {}] server.Server(415): Started @109360ms
2024-11-17T20:35:06,075 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-17T20:35:06,935 WARN  [Thread-673 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/cluster_b9ca40e8-4224-d8ab-7466-d1512b745e25/data/data2/current/BP-1477976244-172.17.0.2-1731875704920/current, will proceed with Du for space computation calculation, 
2024-11-17T20:35:06,935 WARN  [Thread-672 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/cluster_b9ca40e8-4224-d8ab-7466-d1512b745e25/data/data1/current/BP-1477976244-172.17.0.2-1731875704920/current, will proceed with Du for space computation calculation, 
2024-11-17T20:35:06,965 WARN  [Thread-636 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-17T20:35:06,975 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3fd4c7de94af8b9 with lease ID 0x9923bdc54ecc4192: Processing first storage report for DS-5470270f-a72f-4479-b477-b6d1bcd87291 from datanode DatanodeRegistration(127.0.0.1:41145, datanodeUuid=15f63053-16f8-4ae9-8ca1-f284e5d8b564, infoPort=42973, infoSecurePort=0, ipcPort=42067, storageInfo=lv=-57;cid=testClusterID;nsid=991965062;c=1731875704920)
2024-11-17T20:35:06,975 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3fd4c7de94af8b9 with lease ID 0x9923bdc54ecc4192: from storage DS-5470270f-a72f-4479-b477-b6d1bcd87291 node DatanodeRegistration(127.0.0.1:41145, datanodeUuid=15f63053-16f8-4ae9-8ca1-f284e5d8b564, infoPort=42973, infoSecurePort=0, ipcPort=42067, storageInfo=lv=-57;cid=testClusterID;nsid=991965062;c=1731875704920), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-17T20:35:06,976 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3fd4c7de94af8b9 with lease ID 0x9923bdc54ecc4192: Processing first storage report for DS-62053ee5-1ecd-4e13-931c-5fba3a0911aa from datanode DatanodeRegistration(127.0.0.1:41145, datanodeUuid=15f63053-16f8-4ae9-8ca1-f284e5d8b564, infoPort=42973, infoSecurePort=0, ipcPort=42067, storageInfo=lv=-57;cid=testClusterID;nsid=991965062;c=1731875704920)
2024-11-17T20:35:06,976 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3fd4c7de94af8b9 with lease ID 0x9923bdc54ecc4192: from storage DS-62053ee5-1ecd-4e13-931c-5fba3a0911aa node DatanodeRegistration(127.0.0.1:41145, datanodeUuid=15f63053-16f8-4ae9-8ca1-f284e5d8b564, infoPort=42973, infoSecurePort=0, ipcPort=42067, storageInfo=lv=-57;cid=testClusterID;nsid=991965062;c=1731875704920), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-17T20:35:07,136 WARN  [Thread-683 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/cluster_b9ca40e8-4224-d8ab-7466-d1512b745e25/data/data3/current/BP-1477976244-172.17.0.2-1731875704920/current, will proceed with Du for space computation calculation, 
2024-11-17T20:35:07,136 WARN  [Thread-684 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/cluster_b9ca40e8-4224-d8ab-7466-d1512b745e25/data/data4/current/BP-1477976244-172.17.0.2-1731875704920/current, will proceed with Du for space computation calculation, 
2024-11-17T20:35:07,156 WARN  [Thread-659 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-17T20:35:07,159 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6eeaf88adc957b25 with lease ID 0x9923bdc54ecc4193: Processing first storage report for DS-59ce2296-6caf-48fd-a174-2d36e11610d2 from datanode DatanodeRegistration(127.0.0.1:42609, datanodeUuid=6506fff5-03d5-479e-8b59-d4ac07be411d, infoPort=43985, infoSecurePort=0, ipcPort=39959, storageInfo=lv=-57;cid=testClusterID;nsid=991965062;c=1731875704920)
2024-11-17T20:35:07,159 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6eeaf88adc957b25 with lease ID 0x9923bdc54ecc4193: from storage DS-59ce2296-6caf-48fd-a174-2d36e11610d2 node DatanodeRegistration(127.0.0.1:42609, datanodeUuid=6506fff5-03d5-479e-8b59-d4ac07be411d, infoPort=43985, infoSecurePort=0, ipcPort=39959, storageInfo=lv=-57;cid=testClusterID;nsid=991965062;c=1731875704920), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-17T20:35:07,159 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6eeaf88adc957b25 with lease ID 0x9923bdc54ecc4193: Processing first storage report for DS-30f485bb-8766-4446-aa31-1a46b6ed8929 from datanode DatanodeRegistration(127.0.0.1:42609, datanodeUuid=6506fff5-03d5-479e-8b59-d4ac07be411d, infoPort=43985, infoSecurePort=0, ipcPort=39959, storageInfo=lv=-57;cid=testClusterID;nsid=991965062;c=1731875704920)
2024-11-17T20:35:07,159 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6eeaf88adc957b25 with lease ID 0x9923bdc54ecc4193: from storage DS-30f485bb-8766-4446-aa31-1a46b6ed8929 node DatanodeRegistration(127.0.0.1:42609, datanodeUuid=6506fff5-03d5-479e-8b59-d4ac07be411d, infoPort=43985, infoSecurePort=0, ipcPort=39959, storageInfo=lv=-57;cid=testClusterID;nsid=991965062;c=1731875704920), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-17T20:35:07,217 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25
2024-11-17T20:35:07,221 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/cluster_b9ca40e8-4224-d8ab-7466-d1512b745e25/zookeeper_0, clientPort=54994, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/cluster_b9ca40e8-4224-d8ab-7466-d1512b745e25/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/cluster_b9ca40e8-4224-d8ab-7466-d1512b745e25/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-17T20:35:07,224 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=54994
2024-11-17T20:35:07,225 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T20:35:07,226 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T20:35:07,244 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41145 is added to blk_1073741825_1001 (size=7)
2024-11-17T20:35:07,244 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42609 is added to blk_1073741825_1001 (size=7)
2024-11-17T20:35:07,245 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899 with version=8
2024-11-17T20:35:07,246 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/hbase-staging
2024-11-17T20:35:07,247 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/0b0b0ae8de03:0 server-side Connection retries=45
2024-11-17T20:35:07,248 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-17T20:35:07,248 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-17T20:35:07,248 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-17T20:35:07,248 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-17T20:35:07,248 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-17T20:35:07,248 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-17T20:35:07,248 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-17T20:35:07,249 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40559
2024-11-17T20:35:07,250 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:40559 connecting to ZooKeeper ensemble=127.0.0.1:54994
2024-11-17T20:35:07,307 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:405590x0, quorum=127.0.0.1:54994, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-17T20:35:07,308 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:40559-0x1014a8214ff0000 connected
2024-11-17T20:35:07,394 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T20:35:07,396 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T20:35:07,405 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40559-0x1014a8214ff0000, quorum=127.0.0.1:54994, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-17T20:35:07,405 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899, hbase.cluster.distributed=false
2024-11-17T20:35:07,407 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40559-0x1014a8214ff0000, quorum=127.0.0.1:54994, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-17T20:35:07,408 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40559
2024-11-17T20:35:07,408 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40559
2024-11-17T20:35:07,412 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40559
2024-11-17T20:35:07,416 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40559
2024-11-17T20:35:07,416 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40559
2024-11-17T20:35:07,434 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/0b0b0ae8de03:0 server-side Connection retries=45
2024-11-17T20:35:07,434 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-17T20:35:07,435 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-17T20:35:07,435 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-17T20:35:07,435 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-17T20:35:07,435 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-17T20:35:07,435 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-17T20:35:07,435 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-17T20:35:07,436 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33133
2024-11-17T20:35:07,438 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33133 connecting to ZooKeeper ensemble=127.0.0.1:54994
2024-11-17T20:35:07,438 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T20:35:07,440 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T20:35:07,453 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:331330x0, quorum=127.0.0.1:54994, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-17T20:35:07,453 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:331330x0, quorum=127.0.0.1:54994, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-17T20:35:07,454 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33133-0x1014a8214ff0001 connected
2024-11-17T20:35:07,454 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-17T20:35:07,457 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-17T20:35:07,458 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33133-0x1014a8214ff0001, quorum=127.0.0.1:54994, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-17T20:35:07,459 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33133-0x1014a8214ff0001, quorum=127.0.0.1:54994, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-17T20:35:07,467 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33133
2024-11-17T20:35:07,467 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33133
2024-11-17T20:35:07,467 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33133
2024-11-17T20:35:07,472 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33133
2024-11-17T20:35:07,475 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33133
2024-11-17T20:35:07,494 DEBUG [M:0;0b0b0ae8de03:40559 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;0b0b0ae8de03:40559
2024-11-17T20:35:07,494 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/0b0b0ae8de03,40559,1731875707247
2024-11-17T20:35:07,506 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33133-0x1014a8214ff0001, quorum=127.0.0.1:54994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-17T20:35:07,506 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40559-0x1014a8214ff0000, quorum=127.0.0.1:54994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-17T20:35:07,506 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40559-0x1014a8214ff0000, quorum=127.0.0.1:54994, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/0b0b0ae8de03,40559,1731875707247
2024-11-17T20:35:07,520 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40559-0x1014a8214ff0000, quorum=127.0.0.1:54994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:35:07,520 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33133-0x1014a8214ff0001, quorum=127.0.0.1:54994, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-17T20:35:07,520 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33133-0x1014a8214ff0001, quorum=127.0.0.1:54994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:35:07,521 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40559-0x1014a8214ff0000, quorum=127.0.0.1:54994, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-17T20:35:07,522 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/0b0b0ae8de03,40559,1731875707247 from backup master directory
2024-11-17T20:35:07,531 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40559-0x1014a8214ff0000, quorum=127.0.0.1:54994, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/0b0b0ae8de03,40559,1731875707247
2024-11-17T20:35:07,531 WARN  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-17T20:35:07,531 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40559-0x1014a8214ff0000, quorum=127.0.0.1:54994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-17T20:35:07,531 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33133-0x1014a8214ff0001, quorum=127.0.0.1:54994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-17T20:35:07,531 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=0b0b0ae8de03,40559,1731875707247
2024-11-17T20:35:07,538 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/hbase.id] with ID: b6cc5e06-b324-4367-bb93-85fa579351fe
2024-11-17T20:35:07,538 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/.tmp/hbase.id
2024-11-17T20:35:07,562 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42609 is added to blk_1073741826_1002 (size=42)
2024-11-17T20:35:07,563 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41145 is added to blk_1073741826_1002 (size=42)
2024-11-17T20:35:07,567 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/.tmp/hbase.id]:[hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/hbase.id]
2024-11-17T20:35:07,586 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T20:35:07,586 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-17T20:35:07,589 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 3ms.
2024-11-17T20:35:07,600 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40559-0x1014a8214ff0000, quorum=127.0.0.1:54994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:35:07,600 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33133-0x1014a8214ff0001, quorum=127.0.0.1:54994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:35:07,614 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41145 is added to blk_1073741827_1003 (size=196)
2024-11-17T20:35:07,614 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42609 is added to blk_1073741827_1003 (size=196)
2024-11-17T20:35:07,615 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-17T20:35:07,616 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-17T20:35:07,616 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-17T20:35:07,631 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41145 is added to blk_1073741828_1004 (size=1189)
2024-11-17T20:35:07,631 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42609 is added to blk_1073741828_1004 (size=1189)
2024-11-17T20:35:07,633 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/data/master/store
2024-11-17T20:35:07,643 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41145 is added to blk_1073741829_1005 (size=34)
2024-11-17T20:35:07,643 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42609 is added to blk_1073741829_1005 (size=34)
2024-11-17T20:35:07,644 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T20:35:07,645 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-17T20:35:07,645 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T20:35:07,645 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T20:35:07,645 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-17T20:35:07,645 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T20:35:07,645 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T20:35:07,645 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731875707644Disabling compacts and flushes for region at 1731875707644Disabling writes for close at 1731875707645 (+1 ms)Writing region close event to WAL at 1731875707645Closed at 1731875707645
2024-11-17T20:35:07,646 WARN  [master/0b0b0ae8de03:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/data/master/store/.initializing
2024-11-17T20:35:07,646 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247
2024-11-17T20:35:07,650 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0b0b0ae8de03%2C40559%2C1731875707247, suffix=, logDir=hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247, archiveDir=hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/oldWALs, maxLogs=10
2024-11-17T20:35:07,650 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
2024-11-17T20:35:07,658 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
2024-11-17T20:35:07,660 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42973:42973),(127.0.0.1/127.0.0.1:43985:43985)]
2024-11-17T20:35:07,664 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-17T20:35:07,664 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T20:35:07,664 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T20:35:07,665 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T20:35:07,667 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-17T20:35:07,669 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-17T20:35:07,669 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:35:07,670 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T20:35:07,670 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-17T20:35:07,672 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-17T20:35:07,672 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:35:07,673 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-17T20:35:07,673 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-17T20:35:07,675 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-17T20:35:07,675 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:35:07,676 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-17T20:35:07,676 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-17T20:35:07,677 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-17T20:35:07,678 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:35:07,678 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-17T20:35:07,678 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T20:35:07,679 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-17T20:35:07,680 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-17T20:35:07,681 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T20:35:07,681 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T20:35:07,682 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-17T20:35:07,684 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T20:35:07,687 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-17T20:35:07,688 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=781456, jitterRate=-0.006328105926513672}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-17T20:35:07,689 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1731875707665Initializing all the Stores at 1731875707666 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731875707666Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731875707667 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731875707667Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731875707667Cleaning up temporary data from old regions at 1731875707681 (+14 ms)Region opened successfully at 1731875707688 (+7 ms)
2024-11-17T20:35:07,689 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-17T20:35:07,693 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36fb229f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0b0b0ae8de03/172.17.0.2:0
2024-11-17T20:35:07,694 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-17T20:35:07,695 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-17T20:35:07,695 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-17T20:35:07,695 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-17T20:35:07,696 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-17T20:35:07,696 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-17T20:35:07,696 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-17T20:35:07,699 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-17T20:35:07,700 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40559-0x1014a8214ff0000, quorum=127.0.0.1:54994, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-17T20:35:07,710 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-17T20:35:07,710 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-17T20:35:07,711 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40559-0x1014a8214ff0000, quorum=127.0.0.1:54994, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-17T20:35:07,720 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-17T20:35:07,721 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-17T20:35:07,722 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40559-0x1014a8214ff0000, quorum=127.0.0.1:54994, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-17T20:35:07,731 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-17T20:35:07,732 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40559-0x1014a8214ff0000, quorum=127.0.0.1:54994, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-17T20:35:07,741 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-17T20:35:07,745 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40559-0x1014a8214ff0000, quorum=127.0.0.1:54994, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-17T20:35:07,752 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-17T20:35:07,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33133-0x1014a8214ff0001, quorum=127.0.0.1:54994, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-17T20:35:07,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40559-0x1014a8214ff0000, quorum=127.0.0.1:54994, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-17T20:35:07,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33133-0x1014a8214ff0001, quorum=127.0.0.1:54994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:35:07,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40559-0x1014a8214ff0000, quorum=127.0.0.1:54994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:35:07,763 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=0b0b0ae8de03,40559,1731875707247, sessionid=0x1014a8214ff0000, setting cluster-up flag (Was=false)
2024-11-17T20:35:07,783 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33133-0x1014a8214ff0001, quorum=127.0.0.1:54994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:35:07,783 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40559-0x1014a8214ff0000, quorum=127.0.0.1:54994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:35:07,815 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-17T20:35:07,816 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0b0b0ae8de03,40559,1731875707247
2024-11-17T20:35:07,836 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33133-0x1014a8214ff0001, quorum=127.0.0.1:54994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:35:07,836 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40559-0x1014a8214ff0000, quorum=127.0.0.1:54994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:35:07,868 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-17T20:35:07,869 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0b0b0ae8de03,40559,1731875707247
2024-11-17T20:35:07,871 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-17T20:35:07,873 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-17T20:35:07,874 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-17T20:35:07,874 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-17T20:35:07,874 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 0b0b0ae8de03,40559,1731875707247
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-17T20:35:07,876 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/0b0b0ae8de03:0, corePoolSize=5, maxPoolSize=5
2024-11-17T20:35:07,876 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/0b0b0ae8de03:0, corePoolSize=5, maxPoolSize=5
2024-11-17T20:35:07,876 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/0b0b0ae8de03:0, corePoolSize=5, maxPoolSize=5
2024-11-17T20:35:07,876 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/0b0b0ae8de03:0, corePoolSize=5, maxPoolSize=5
2024-11-17T20:35:07,877 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/0b0b0ae8de03:0, corePoolSize=10, maxPoolSize=10
2024-11-17T20:35:07,877 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:35:07,877 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/0b0b0ae8de03:0, corePoolSize=2, maxPoolSize=2
2024-11-17T20:35:07,877 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:35:07,880 INFO  [RS:0;0b0b0ae8de03:33133 {}] regionserver.HRegionServer(746): ClusterId : b6cc5e06-b324-4367-bb93-85fa579351fe
2024-11-17T20:35:07,880 DEBUG [RS:0;0b0b0ae8de03:33133 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-17T20:35:07,886 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-17T20:35:07,886 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-17T20:35:07,888 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:35:07,888 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-17T20:35:07,900 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731875737900
2024-11-17T20:35:07,901 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-17T20:35:07,901 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-17T20:35:07,901 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-17T20:35:07,901 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-17T20:35:07,901 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-17T20:35:07,901 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-17T20:35:07,902 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-17T20:35:07,904 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-17T20:35:07,905 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-17T20:35:07,905 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-17T20:35:07,911 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42609 is added to blk_1073741831_1007 (size=1321)
2024-11-17T20:35:07,911 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41145 is added to blk_1073741831_1007 (size=1321)
2024-11-17T20:35:07,916 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-17T20:35:07,916 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-17T20:35:07,920 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/0b0b0ae8de03:0:becomeActiveMaster-HFileCleaner.large.0-1731875707916,5,FailOnTimeoutGroup]
2024-11-17T20:35:07,920 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/0b0b0ae8de03:0:becomeActiveMaster-HFileCleaner.small.0-1731875707920,5,FailOnTimeoutGroup]
2024-11-17T20:35:07,921 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-17T20:35:07,921 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-17T20:35:07,921 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-17T20:35:07,921 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-17T20:35:07,938 DEBUG [RS:0;0b0b0ae8de03:33133 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-17T20:35:07,938 DEBUG [RS:0;0b0b0ae8de03:33133 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-17T20:35:08,032 DEBUG [RS:0;0b0b0ae8de03:33133 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-17T20:35:08,032 DEBUG [RS:0;0b0b0ae8de03:33133 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4294e837, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0b0b0ae8de03/172.17.0.2:0
2024-11-17T20:35:08,051 DEBUG [RS:0;0b0b0ae8de03:33133 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;0b0b0ae8de03:33133
2024-11-17T20:35:08,051 INFO  [RS:0;0b0b0ae8de03:33133 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-17T20:35:08,051 INFO  [RS:0;0b0b0ae8de03:33133 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-17T20:35:08,051 DEBUG [RS:0;0b0b0ae8de03:33133 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-17T20:35:08,052 INFO  [RS:0;0b0b0ae8de03:33133 {}] regionserver.HRegionServer(2659): reportForDuty to master=0b0b0ae8de03,40559,1731875707247 with port=33133, startcode=1731875707434
2024-11-17T20:35:08,053 DEBUG [RS:0;0b0b0ae8de03:33133 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-17T20:35:08,056 INFO  [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46623, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService
2024-11-17T20:35:08,057 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40559 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 0b0b0ae8de03,33133,1731875707434
2024-11-17T20:35:08,057 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40559 {}] master.ServerManager(517): Registering regionserver=0b0b0ae8de03,33133,1731875707434
2024-11-17T20:35:08,059 DEBUG [RS:0;0b0b0ae8de03:33133 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899
2024-11-17T20:35:08,059 DEBUG [RS:0;0b0b0ae8de03:33133 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:33551
2024-11-17T20:35:08,059 DEBUG [RS:0;0b0b0ae8de03:33133 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-17T20:35:08,084 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40559-0x1014a8214ff0000, quorum=127.0.0.1:54994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-17T20:35:08,085 DEBUG [RS:0;0b0b0ae8de03:33133 {}] zookeeper.ZKUtil(111): regionserver:33133-0x1014a8214ff0001, quorum=127.0.0.1:54994, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/0b0b0ae8de03,33133,1731875707434
2024-11-17T20:35:08,085 WARN  [RS:0;0b0b0ae8de03:33133 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-17T20:35:08,086 INFO  [RS:0;0b0b0ae8de03:33133 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-17T20:35:08,086 DEBUG [RS:0;0b0b0ae8de03:33133 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434
2024-11-17T20:35:08,089 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [0b0b0ae8de03,33133,1731875707434]
2024-11-17T20:35:08,091 INFO  [RS:0;0b0b0ae8de03:33133 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-17T20:35:08,108 INFO  [RS:0;0b0b0ae8de03:33133 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-17T20:35:08,124 INFO  [RS:0;0b0b0ae8de03:33133 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-17T20:35:08,125 INFO  [RS:0;0b0b0ae8de03:33133 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-17T20:35:08,125 INFO  [RS:0;0b0b0ae8de03:33133 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-17T20:35:08,126 INFO  [RS:0;0b0b0ae8de03:33133 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-17T20:35:08,127 INFO  [RS:0;0b0b0ae8de03:33133 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-17T20:35:08,127 DEBUG [RS:0;0b0b0ae8de03:33133 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:35:08,127 DEBUG [RS:0;0b0b0ae8de03:33133 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:35:08,127 DEBUG [RS:0;0b0b0ae8de03:33133 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:35:08,127 DEBUG [RS:0;0b0b0ae8de03:33133 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:35:08,127 DEBUG [RS:0;0b0b0ae8de03:33133 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:35:08,128 DEBUG [RS:0;0b0b0ae8de03:33133 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/0b0b0ae8de03:0, corePoolSize=2, maxPoolSize=2
2024-11-17T20:35:08,128 DEBUG [RS:0;0b0b0ae8de03:33133 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:35:08,128 DEBUG [RS:0;0b0b0ae8de03:33133 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:35:08,128 DEBUG [RS:0;0b0b0ae8de03:33133 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:35:08,128 DEBUG [RS:0;0b0b0ae8de03:33133 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:35:08,128 DEBUG [RS:0;0b0b0ae8de03:33133 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:35:08,128 DEBUG [RS:0;0b0b0ae8de03:33133 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:35:08,128 DEBUG [RS:0;0b0b0ae8de03:33133 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/0b0b0ae8de03:0, corePoolSize=3, maxPoolSize=3
2024-11-17T20:35:08,128 DEBUG [RS:0;0b0b0ae8de03:33133 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0, corePoolSize=3, maxPoolSize=3
2024-11-17T20:35:08,129 INFO  [RS:0;0b0b0ae8de03:33133 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-17T20:35:08,129 INFO  [RS:0;0b0b0ae8de03:33133 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-17T20:35:08,129 INFO  [RS:0;0b0b0ae8de03:33133 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-17T20:35:08,129 INFO  [RS:0;0b0b0ae8de03:33133 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-17T20:35:08,129 INFO  [RS:0;0b0b0ae8de03:33133 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-17T20:35:08,129 INFO  [RS:0;0b0b0ae8de03:33133 {}] hbase.ChoreService(168): Chore ScheduledChore name=0b0b0ae8de03,33133,1731875707434-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-17T20:35:08,151 INFO  [RS:0;0b0b0ae8de03:33133 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-17T20:35:08,152 INFO  [RS:0;0b0b0ae8de03:33133 {}] hbase.ChoreService(168): Chore ScheduledChore name=0b0b0ae8de03,33133,1731875707434-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-17T20:35:08,152 INFO  [RS:0;0b0b0ae8de03:33133 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T20:35:08,152 INFO  [RS:0;0b0b0ae8de03:33133 {}] regionserver.Replication(171): 0b0b0ae8de03,33133,1731875707434 started
2024-11-17T20:35:08,173 INFO  [RS:0;0b0b0ae8de03:33133 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T20:35:08,173 INFO  [RS:0;0b0b0ae8de03:33133 {}] regionserver.HRegionServer(1482): Serving as 0b0b0ae8de03,33133,1731875707434, RpcServer on 0b0b0ae8de03/172.17.0.2:33133, sessionid=0x1014a8214ff0001
2024-11-17T20:35:08,173 DEBUG [RS:0;0b0b0ae8de03:33133 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-17T20:35:08,174 DEBUG [RS:0;0b0b0ae8de03:33133 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 0b0b0ae8de03,33133,1731875707434
2024-11-17T20:35:08,174 DEBUG [RS:0;0b0b0ae8de03:33133 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0b0b0ae8de03,33133,1731875707434'
2024-11-17T20:35:08,174 DEBUG [RS:0;0b0b0ae8de03:33133 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-17T20:35:08,174 DEBUG [RS:0;0b0b0ae8de03:33133 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-17T20:35:08,175 DEBUG [RS:0;0b0b0ae8de03:33133 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-17T20:35:08,175 DEBUG [RS:0;0b0b0ae8de03:33133 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-17T20:35:08,175 DEBUG [RS:0;0b0b0ae8de03:33133 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 0b0b0ae8de03,33133,1731875707434
2024-11-17T20:35:08,175 DEBUG [RS:0;0b0b0ae8de03:33133 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0b0b0ae8de03,33133,1731875707434'
2024-11-17T20:35:08,175 DEBUG [RS:0;0b0b0ae8de03:33133 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-17T20:35:08,177 DEBUG [RS:0;0b0b0ae8de03:33133 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-17T20:35:08,177 DEBUG [RS:0;0b0b0ae8de03:33133 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-17T20:35:08,177 INFO  [RS:0;0b0b0ae8de03:33133 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-17T20:35:08,178 INFO  [RS:0;0b0b0ae8de03:33133 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-17T20:35:08,280 INFO  [RS:0;0b0b0ae8de03:33133 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0b0b0ae8de03%2C33133%2C1731875707434, suffix=, logDir=hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434, archiveDir=hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/oldWALs, maxLogs=32
2024-11-17T20:35:08,281 INFO  [RS:0;0b0b0ae8de03:33133 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0b0b0ae8de03%2C33133%2C1731875707434.1731875708280
2024-11-17T20:35:08,290 INFO  [RS:0;0b0b0ae8de03:33133 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.1731875708280
2024-11-17T20:35:08,291 DEBUG [RS:0;0b0b0ae8de03:33133 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43985:43985),(127.0.0.1/127.0.0.1:42973:42973)]
2024-11-17T20:35:08,313 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-17T20:35:08,313 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899
2024-11-17T20:35:08,327 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41145 is added to blk_1073741833_1009 (size=32)
2024-11-17T20:35:08,327 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42609 is added to blk_1073741833_1009 (size=32)
2024-11-17T20:35:08,328 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T20:35:08,336 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-17T20:35:08,354 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-17T20:35:08,354 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:35:08,355 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T20:35:08,355 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-17T20:35:08,356 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-17T20:35:08,356 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:35:08,357 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T20:35:08,357 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-17T20:35:08,358 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-17T20:35:08,358 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:35:08,369 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T20:35:08,369 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-17T20:35:08,371 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-17T20:35:08,371 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:35:08,372 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T20:35:08,372 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-17T20:35:08,373 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/hbase/meta/1588230740
2024-11-17T20:35:08,373 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/hbase/meta/1588230740
2024-11-17T20:35:08,374 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-17T20:35:08,374 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-17T20:35:08,375 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-17T20:35:08,376 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-17T20:35:08,381 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-17T20:35:08,382 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=835995, jitterRate=0.06302273273468018}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-17T20:35:08,382 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1731875708328Initializing all the Stores at 1731875708329 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731875708329Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731875708336 (+7 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731875708336Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731875708336Cleaning up temporary data from old regions at 1731875708374 (+38 ms)Region opened successfully at 1731875708382 (+8 ms)
2024-11-17T20:35:08,383 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-17T20:35:08,383 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-17T20:35:08,383 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-17T20:35:08,383 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-17T20:35:08,383 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-17T20:35:08,384 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-17T20:35:08,384 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731875708383Disabling compacts and flushes for region at 1731875708383Disabling writes for close at 1731875708383Writing region close event to WAL at 1731875708384 (+1 ms)Closed at 1731875708384
2024-11-17T20:35:08,385 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-17T20:35:08,385 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-17T20:35:08,386 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-17T20:35:08,388 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-17T20:35:08,389 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-17T20:35:08,539 DEBUG [0b0b0ae8de03:40559 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-17T20:35:08,540 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=0b0b0ae8de03,33133,1731875707434
2024-11-17T20:35:08,541 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0b0b0ae8de03,33133,1731875707434, state=OPENING
2024-11-17T20:35:08,646 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-17T20:35:08,657 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40559-0x1014a8214ff0000, quorum=127.0.0.1:54994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:35:08,657 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33133-0x1014a8214ff0001, quorum=127.0.0.1:54994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:35:08,658 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-17T20:35:08,658 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-17T20:35:08,658 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-17T20:35:08,658 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=0b0b0ae8de03,33133,1731875707434}]
2024-11-17T20:35:08,812 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-17T20:35:08,814 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53173, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-17T20:35:08,819 INFO  [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-17T20:35:08,819 INFO  [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-17T20:35:08,821 INFO  [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0b0b0ae8de03%2C33133%2C1731875707434.meta, suffix=.meta, logDir=hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434, archiveDir=hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/oldWALs, maxLogs=32
2024-11-17T20:35:08,823 INFO  [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
2024-11-17T20:35:08,832 INFO  [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
2024-11-17T20:35:08,856 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42973:42973),(127.0.0.1/127.0.0.1:43985:43985)]
2024-11-17T20:35:08,864 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-17T20:35:08,865 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-17T20:35:08,865 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-17T20:35:08,865 INFO  [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-17T20:35:08,865 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-17T20:35:08,865 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T20:35:08,865 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-17T20:35:08,865 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-17T20:35:08,867 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-17T20:35:08,868 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-17T20:35:08,868 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:35:08,869 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T20:35:08,869 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-17T20:35:08,871 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-17T20:35:08,871 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:35:08,872 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T20:35:08,872 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-17T20:35:08,873 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-17T20:35:08,873 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:35:08,874 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T20:35:08,874 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-17T20:35:08,875 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-17T20:35:08,875 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:35:08,875 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T20:35:08,876 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-17T20:35:08,876 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/hbase/meta/1588230740
2024-11-17T20:35:08,877 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/hbase/meta/1588230740
2024-11-17T20:35:08,879 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-17T20:35:08,879 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-17T20:35:08,880 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-17T20:35:08,881 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-17T20:35:08,882 INFO  [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=808564, jitterRate=0.02814297378063202}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-17T20:35:08,883 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-17T20:35:08,884 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1731875708866Writing region info on filesystem at 1731875708866Initializing all the Stores at 1731875708867 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731875708867Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731875708867Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731875708867Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731875708867Cleaning up temporary data from old regions at 1731875708879 (+12 ms)Running coprocessor post-open hooks at 1731875708883 (+4 ms)Region opened successfully at 1731875708884 (+1 ms)
2024-11-17T20:35:08,885 INFO  [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731875708812
2024-11-17T20:35:08,888 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-17T20:35:08,888 INFO  [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-17T20:35:08,890 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=0b0b0ae8de03,33133,1731875707434
2024-11-17T20:35:08,891 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0b0b0ae8de03,33133,1731875707434, state=OPEN
2024-11-17T20:35:08,929 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40559-0x1014a8214ff0000, quorum=127.0.0.1:54994, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-17T20:35:08,929 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33133-0x1014a8214ff0001, quorum=127.0.0.1:54994, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-17T20:35:08,929 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=0b0b0ae8de03,33133,1731875707434
2024-11-17T20:35:08,929 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-17T20:35:08,929 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-17T20:35:08,933 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-17T20:35:08,933 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=0b0b0ae8de03,33133,1731875707434 in 271 msec
2024-11-17T20:35:08,937 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-17T20:35:08,937 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 547 msec
2024-11-17T20:35:08,938 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-17T20:35:08,938 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-17T20:35:08,940 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-17T20:35:08,940 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,33133,1731875707434, seqNum=-1]
2024-11-17T20:35:08,941 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-17T20:35:08,942 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44189, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-17T20:35:08,951 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0770 sec
2024-11-17T20:35:08,951 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731875708951, completionTime=-1
2024-11-17T20:35:08,951 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-17T20:35:08,951 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster...
2024-11-17T20:35:08,954 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1
2024-11-17T20:35:08,954 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731875768954
2024-11-17T20:35:08,954 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731875828954
2024-11-17T20:35:08,954 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 3 msec
2024-11-17T20:35:08,955 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0b0b0ae8de03,40559,1731875707247-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-17T20:35:08,955 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0b0b0ae8de03,40559,1731875707247-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T20:35:08,955 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0b0b0ae8de03,40559,1731875707247-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T20:35:08,955 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-0b0b0ae8de03:40559, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T20:35:08,955 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-17T20:35:08,956 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-17T20:35:08,958 DEBUG [master/0b0b0ae8de03:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-17T20:35:08,960 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.429sec
2024-11-17T20:35:08,960 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-17T20:35:08,960 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-17T20:35:08,960 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-17T20:35:08,960 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-17T20:35:08,960 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-17T20:35:08,960 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0b0b0ae8de03,40559,1731875707247-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-17T20:35:08,960 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0b0b0ae8de03,40559,1731875707247-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-17T20:35:08,963 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-17T20:35:08,963 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-17T20:35:08,963 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0b0b0ae8de03,40559,1731875707247-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T20:35:08,981 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e032178, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-17T20:35:08,981 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b0b0ae8de03,40559,-1 for getting cluster id
2024-11-17T20:35:08,981 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-17T20:35:08,983 DEBUG [HMaster-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b6cc5e06-b324-4367-bb93-85fa579351fe'
2024-11-17T20:35:08,983 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-17T20:35:08,983 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b6cc5e06-b324-4367-bb93-85fa579351fe"

2024-11-17T20:35:08,984 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f2f8d94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-17T20:35:08,984 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b0b0ae8de03,40559,-1]
2024-11-17T20:35:08,984 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-17T20:35:08,985 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T20:35:08,986 INFO  [HMaster-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38450, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-17T20:35:08,988 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60d7890e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-17T20:35:08,988 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-17T20:35:08,989 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,33133,1731875707434, seqNum=-1]
2024-11-17T20:35:08,990 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-17T20:35:08,992 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57132, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-17T20:35:08,994 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=0b0b0ae8de03,40559,1731875707247
2024-11-17T20:35:08,994 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T20:35:08,997 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-17T20:35:09,021 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/0b0b0ae8de03:0 server-side Connection retries=45
2024-11-17T20:35:09,021 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-17T20:35:09,021 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-17T20:35:09,021 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-17T20:35:09,021 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-17T20:35:09,021 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-17T20:35:09,021 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-17T20:35:09,022 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-17T20:35:09,022 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37509
2024-11-17T20:35:09,025 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37509 connecting to ZooKeeper ensemble=127.0.0.1:54994
2024-11-17T20:35:09,026 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T20:35:09,028 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T20:35:09,053 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:375090x0, quorum=127.0.0.1:54994, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-17T20:35:09,053 DEBUG [pool-381-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT
2024-11-17T20:35:09,053 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:375090x0, quorum=127.0.0.1:54994, baseZNode=/hbase Set watcher on existing znode=/hbase/running
2024-11-17T20:35:09,053 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37509-0x1014a8214ff0002 connected
2024-11-17T20:35:09,054 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-17T20:35:09,055 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-17T20:35:09,055 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:37509-0x1014a8214ff0002, quorum=127.0.0.1:54994, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-17T20:35:09,057 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37509-0x1014a8214ff0002, quorum=127.0.0.1:54994, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-17T20:35:09,064 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37509
2024-11-17T20:35:09,065 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37509
2024-11-17T20:35:09,072 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37509
2024-11-17T20:35:09,084 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37509
2024-11-17T20:35:09,088 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37509
2024-11-17T20:35:09,090 INFO  [RS:1;0b0b0ae8de03:37509 {}] regionserver.HRegionServer(746): ClusterId : b6cc5e06-b324-4367-bb93-85fa579351fe
2024-11-17T20:35:09,090 DEBUG [RS:1;0b0b0ae8de03:37509 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-17T20:35:09,100 DEBUG [RS:1;0b0b0ae8de03:37509 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-17T20:35:09,100 DEBUG [RS:1;0b0b0ae8de03:37509 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-17T20:35:09,111 DEBUG [RS:1;0b0b0ae8de03:37509 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-17T20:35:09,111 DEBUG [RS:1;0b0b0ae8de03:37509 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4737b6ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0b0b0ae8de03/172.17.0.2:0
2024-11-17T20:35:09,126 DEBUG [RS:1;0b0b0ae8de03:37509 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;0b0b0ae8de03:37509
2024-11-17T20:35:09,126 INFO  [RS:1;0b0b0ae8de03:37509 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-17T20:35:09,126 INFO  [RS:1;0b0b0ae8de03:37509 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-17T20:35:09,126 DEBUG [RS:1;0b0b0ae8de03:37509 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-17T20:35:09,127 INFO  [RS:1;0b0b0ae8de03:37509 {}] regionserver.HRegionServer(2659): reportForDuty to master=0b0b0ae8de03,40559,1731875707247 with port=37509, startcode=1731875709020
2024-11-17T20:35:09,127 DEBUG [RS:1;0b0b0ae8de03:37509 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-17T20:35:09,129 INFO  [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46193, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.3 (auth:SIMPLE), service=RegionServerStatusService
2024-11-17T20:35:09,129 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40559 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 0b0b0ae8de03,37509,1731875709020
2024-11-17T20:35:09,129 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40559 {}] master.ServerManager(517): Registering regionserver=0b0b0ae8de03,37509,1731875709020
2024-11-17T20:35:09,131 DEBUG [RS:1;0b0b0ae8de03:37509 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899
2024-11-17T20:35:09,131 DEBUG [RS:1;0b0b0ae8de03:37509 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:33551
2024-11-17T20:35:09,131 DEBUG [RS:1;0b0b0ae8de03:37509 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-17T20:35:09,141 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40559-0x1014a8214ff0000, quorum=127.0.0.1:54994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-17T20:35:09,142 DEBUG [RS:1;0b0b0ae8de03:37509 {}] zookeeper.ZKUtil(111): regionserver:37509-0x1014a8214ff0002, quorum=127.0.0.1:54994, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/0b0b0ae8de03,37509,1731875709020
2024-11-17T20:35:09,142 WARN  [RS:1;0b0b0ae8de03:37509 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-17T20:35:09,142 INFO  [RS:1;0b0b0ae8de03:37509 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-17T20:35:09,142 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [0b0b0ae8de03,37509,1731875709020]
2024-11-17T20:35:09,142 DEBUG [RS:1;0b0b0ae8de03:37509 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020
2024-11-17T20:35:09,146 INFO  [RS:1;0b0b0ae8de03:37509 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-17T20:35:09,148 INFO  [RS:1;0b0b0ae8de03:37509 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-17T20:35:09,148 INFO  [RS:1;0b0b0ae8de03:37509 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-17T20:35:09,148 INFO  [RS:1;0b0b0ae8de03:37509 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-17T20:35:09,149 INFO  [RS:1;0b0b0ae8de03:37509 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-17T20:35:09,149 INFO  [RS:1;0b0b0ae8de03:37509 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-17T20:35:09,150 INFO  [RS:1;0b0b0ae8de03:37509 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-17T20:35:09,150 DEBUG [RS:1;0b0b0ae8de03:37509 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:35:09,150 DEBUG [RS:1;0b0b0ae8de03:37509 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:35:09,150 DEBUG [RS:1;0b0b0ae8de03:37509 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:35:09,150 DEBUG [RS:1;0b0b0ae8de03:37509 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:35:09,150 DEBUG [RS:1;0b0b0ae8de03:37509 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:35:09,150 DEBUG [RS:1;0b0b0ae8de03:37509 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/0b0b0ae8de03:0, corePoolSize=2, maxPoolSize=2
2024-11-17T20:35:09,150 DEBUG [RS:1;0b0b0ae8de03:37509 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:35:09,150 DEBUG [RS:1;0b0b0ae8de03:37509 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:35:09,150 DEBUG [RS:1;0b0b0ae8de03:37509 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:35:09,150 DEBUG [RS:1;0b0b0ae8de03:37509 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:35:09,150 DEBUG [RS:1;0b0b0ae8de03:37509 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:35:09,150 DEBUG [RS:1;0b0b0ae8de03:37509 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:35:09,151 DEBUG [RS:1;0b0b0ae8de03:37509 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/0b0b0ae8de03:0, corePoolSize=3, maxPoolSize=3
2024-11-17T20:35:09,151 DEBUG [RS:1;0b0b0ae8de03:37509 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0, corePoolSize=3, maxPoolSize=3
2024-11-17T20:35:09,151 INFO  [RS:1;0b0b0ae8de03:37509 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-17T20:35:09,151 INFO  [RS:1;0b0b0ae8de03:37509 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-17T20:35:09,151 INFO  [RS:1;0b0b0ae8de03:37509 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-17T20:35:09,151 INFO  [RS:1;0b0b0ae8de03:37509 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-17T20:35:09,152 INFO  [RS:1;0b0b0ae8de03:37509 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-17T20:35:09,152 INFO  [RS:1;0b0b0ae8de03:37509 {}] hbase.ChoreService(168): Chore ScheduledChore name=0b0b0ae8de03,37509,1731875709020-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-17T20:35:09,170 INFO  [RS:1;0b0b0ae8de03:37509 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-17T20:35:09,170 INFO  [RS:1;0b0b0ae8de03:37509 {}] hbase.ChoreService(168): Chore ScheduledChore name=0b0b0ae8de03,37509,1731875709020-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-17T20:35:09,170 INFO  [RS:1;0b0b0ae8de03:37509 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T20:35:09,170 INFO  [RS:1;0b0b0ae8de03:37509 {}] regionserver.Replication(171): 0b0b0ae8de03,37509,1731875709020 started
2024-11-17T20:35:09,187 INFO  [RS:1;0b0b0ae8de03:37509 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T20:35:09,187 INFO  [RS:1;0b0b0ae8de03:37509 {}] regionserver.HRegionServer(1482): Serving as 0b0b0ae8de03,37509,1731875709020, RpcServer on 0b0b0ae8de03/172.17.0.2:37509, sessionid=0x1014a8214ff0002
2024-11-17T20:35:09,187 DEBUG [RS:1;0b0b0ae8de03:37509 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-17T20:35:09,187 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(2882): Started new server=Thread[RS:1;0b0b0ae8de03:37509,5,FailOnTimeoutGroup]
2024-11-17T20:35:09,187 DEBUG [RS:1;0b0b0ae8de03:37509 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 0b0b0ae8de03,37509,1731875709020
2024-11-17T20:35:09,187 DEBUG [RS:1;0b0b0ae8de03:37509 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0b0b0ae8de03,37509,1731875709020'
2024-11-17T20:35:09,187 DEBUG [RS:1;0b0b0ae8de03:37509 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-17T20:35:09,188 INFO  [Time-limited test {}] wal.TestLogRolling(207): Replication=2
2024-11-17T20:35:09,188 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-17T20:35:09,188 DEBUG [RS:1;0b0b0ae8de03:37509 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-17T20:35:09,189 DEBUG [RS:1;0b0b0ae8de03:37509 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-17T20:35:09,189 DEBUG [RS:1;0b0b0ae8de03:37509 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-17T20:35:09,189 DEBUG [RS:1;0b0b0ae8de03:37509 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 0b0b0ae8de03,37509,1731875709020
2024-11-17T20:35:09,189 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncConnectionImpl(321): The fetched master address is 0b0b0ae8de03,40559,1731875707247
2024-11-17T20:35:09,189 DEBUG [RS:1;0b0b0ae8de03:37509 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0b0b0ae8de03,37509,1731875709020'
2024-11-17T20:35:09,189 DEBUG [RS:1;0b0b0ae8de03:37509 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-17T20:35:09,189 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@29e99e4f
2024-11-17T20:35:09,189 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-17T20:35:09,190 DEBUG [RS:1;0b0b0ae8de03:37509 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-17T20:35:09,190 DEBUG [RS:1;0b0b0ae8de03:37509 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-17T20:35:09,190 INFO  [RS:1;0b0b0ae8de03:37509 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-17T20:35:09,190 INFO  [RS:1;0b0b0ae8de03:37509 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-17T20:35:09,192 INFO  [HMaster-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38462, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-17T20:35:09,193 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40559 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-11-17T20:35:09,193 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40559 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-11-17T20:35:09,193 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40559 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRollOnDatanodeDeath', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-17T20:35:09,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40559 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath
2024-11-17T20:35:09,196 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_PRE_OPERATION
2024-11-17T20:35:09,196 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:35:09,196 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40559 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRollOnDatanodeDeath"
 procId is: 4
2024-11-17T20:35:09,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40559 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-17T20:35:09,198 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-17T20:35:09,217 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42609 is added to blk_1073741835_1011 (size=393)
2024-11-17T20:35:09,217 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41145 is added to blk_1073741835_1011 (size=393)
2024-11-17T20:35:09,293 INFO  [RS:1;0b0b0ae8de03:37509 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0b0b0ae8de03%2C37509%2C1731875709020, suffix=, logDir=hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020, archiveDir=hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/oldWALs, maxLogs=32
2024-11-17T20:35:09,295 INFO  [RS:1;0b0b0ae8de03:37509 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
2024-11-17T20:35:09,318 INFO  [RS:1;0b0b0ae8de03:37509 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
2024-11-17T20:35:09,328 DEBUG [RS:1;0b0b0ae8de03:37509 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43985:43985),(127.0.0.1/127.0.0.1:42973:42973)]
2024-11-17T20:35:09,620 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6ea106502026e091c4a7f9fa49641279, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1731875709192.6ea106502026e091c4a7f9fa49641279.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnDatanodeDeath', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899
2024-11-17T20:35:09,629 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42609 is added to blk_1073741837_1013 (size=76)
2024-11-17T20:35:09,630 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41145 is added to blk_1073741837_1013 (size=76)
2024-11-17T20:35:09,631 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1731875709192.6ea106502026e091c4a7f9fa49641279.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T20:35:09,631 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1722): Closing 6ea106502026e091c4a7f9fa49641279, disabling compactions & flushes
2024-11-17T20:35:09,631 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1731875709192.6ea106502026e091c4a7f9fa49641279.
2024-11-17T20:35:09,631 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1731875709192.6ea106502026e091c4a7f9fa49641279.
2024-11-17T20:35:09,631 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1731875709192.6ea106502026e091c4a7f9fa49641279. after waiting 0 ms
2024-11-17T20:35:09,631 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1731875709192.6ea106502026e091c4a7f9fa49641279.
2024-11-17T20:35:09,631 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1731875709192.6ea106502026e091c4a7f9fa49641279.
2024-11-17T20:35:09,631 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6ea106502026e091c4a7f9fa49641279:
Waiting for close lock at 1731875709631Disabling compacts and flushes for region at 1731875709631Disabling writes for close at 1731875709631Writing region close event to WAL at 1731875709631Closed at 1731875709631
2024-11-17T20:35:09,633 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ADD_TO_META
2024-11-17T20:35:09,634 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnDatanodeDeath,,1731875709192.6ea106502026e091c4a7f9fa49641279.","families":{"info":[{"qualifier":"regioninfo","vlen":75,"tag":[],"timestamp":"1731875709633"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731875709633"}]},"ts":"1731875709633"}
2024-11-17T20:35:09,637 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-17T20:35:09,639 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-17T20:35:09,640 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731875709639"}]},"ts":"1731875709639"}
2024-11-17T20:35:09,642 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLING in hbase:meta
2024-11-17T20:35:09,647 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {0b0b0ae8de03=0} racks are {/default-rack=0}
2024-11-17T20:35:09,652 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions
2024-11-17T20:35:09,653 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions
2024-11-17T20:35:09,653 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0
2024-11-17T20:35:09,653 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0
2024-11-17T20:35:09,653 INFO  [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0
2024-11-17T20:35:09,653 INFO  [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0
2024-11-17T20:35:09,653 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1
2024-11-17T20:35:09,654 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=6ea106502026e091c4a7f9fa49641279, ASSIGN}]
2024-11-17T20:35:09,656 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=6ea106502026e091c4a7f9fa49641279, ASSIGN
2024-11-17T20:35:09,658 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=6ea106502026e091c4a7f9fa49641279, ASSIGN; state=OFFLINE, location=0b0b0ae8de03,33133,1731875707434; forceNewPlan=false, retain=false
2024-11-17T20:35:09,809 INFO  [0b0b0ae8de03:40559 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 
2024-11-17T20:35:09,809 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=6ea106502026e091c4a7f9fa49641279, regionState=OPENING, regionLocation=0b0b0ae8de03,33133,1731875707434
2024-11-17T20:35:09,812 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=6ea106502026e091c4a7f9fa49641279, ASSIGN because future has completed
2024-11-17T20:35:09,813 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6ea106502026e091c4a7f9fa49641279, server=0b0b0ae8de03,33133,1731875707434}]
2024-11-17T20:35:09,973 INFO  [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRollOnDatanodeDeath,,1731875709192.6ea106502026e091c4a7f9fa49641279.
2024-11-17T20:35:09,973 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 6ea106502026e091c4a7f9fa49641279, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1731875709192.6ea106502026e091c4a7f9fa49641279.', STARTKEY => '', ENDKEY => ''}
2024-11-17T20:35:09,974 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnDatanodeDeath 6ea106502026e091c4a7f9fa49641279
2024-11-17T20:35:09,974 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1731875709192.6ea106502026e091c4a7f9fa49641279.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T20:35:09,975 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 6ea106502026e091c4a7f9fa49641279
2024-11-17T20:35:09,975 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 6ea106502026e091c4a7f9fa49641279
2024-11-17T20:35:09,977 INFO  [StoreOpener-6ea106502026e091c4a7f9fa49641279-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 6ea106502026e091c4a7f9fa49641279 
2024-11-17T20:35:09,979 INFO  [StoreOpener-6ea106502026e091c4a7f9fa49641279-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6ea106502026e091c4a7f9fa49641279 columnFamilyName info
2024-11-17T20:35:09,979 DEBUG [StoreOpener-6ea106502026e091c4a7f9fa49641279-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:35:09,980 INFO  [StoreOpener-6ea106502026e091c4a7f9fa49641279-1 {}] regionserver.HStore(327): Store=6ea106502026e091c4a7f9fa49641279/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-17T20:35:09,980 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 6ea106502026e091c4a7f9fa49641279
2024-11-17T20:35:09,981 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279
2024-11-17T20:35:09,982 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279
2024-11-17T20:35:09,982 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 6ea106502026e091c4a7f9fa49641279
2024-11-17T20:35:09,983 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 6ea106502026e091c4a7f9fa49641279
2024-11-17T20:35:09,985 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 6ea106502026e091c4a7f9fa49641279
2024-11-17T20:35:09,988 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-17T20:35:09,989 INFO  [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 6ea106502026e091c4a7f9fa49641279; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=749604, jitterRate=-0.04683029651641846}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-17T20:35:09,989 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6ea106502026e091c4a7f9fa49641279
2024-11-17T20:35:09,990 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 6ea106502026e091c4a7f9fa49641279:
Running coprocessor pre-open hook at 1731875709975Writing region info on filesystem at 1731875709975Initializing all the Stores at 1731875709976 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731875709976Cleaning up temporary data from old regions at 1731875709983 (+7 ms)Running coprocessor post-open hooks at 1731875709989 (+6 ms)Region opened successfully at 1731875709990 (+1 ms)
2024-11-17T20:35:09,991 INFO  [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRollOnDatanodeDeath,,1731875709192.6ea106502026e091c4a7f9fa49641279., pid=6, masterSystemTime=1731875709966
2024-11-17T20:35:09,994 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRollOnDatanodeDeath,,1731875709192.6ea106502026e091c4a7f9fa49641279.
2024-11-17T20:35:09,994 INFO  [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRollOnDatanodeDeath,,1731875709192.6ea106502026e091c4a7f9fa49641279.
2024-11-17T20:35:09,995 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=6ea106502026e091c4a7f9fa49641279, regionState=OPEN, openSeqNum=2, regionLocation=0b0b0ae8de03,33133,1731875707434
2024-11-17T20:35:09,998 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6ea106502026e091c4a7f9fa49641279, server=0b0b0ae8de03,33133,1731875707434 because future has completed
2024-11-17T20:35:10,003 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-17T20:35:10,003 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 6ea106502026e091c4a7f9fa49641279, server=0b0b0ae8de03,33133,1731875707434 in 187 msec
2024-11-17T20:35:10,006 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-17T20:35:10,006 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=6ea106502026e091c4a7f9fa49641279, ASSIGN in 349 msec
2024-11-17T20:35:10,008 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-17T20:35:10,008 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731875710008"}]},"ts":"1731875710008"}
2024-11-17T20:35:10,011 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLED in hbase:meta
2024-11-17T20:35:10,012 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_POST_OPERATION
2024-11-17T20:35:10,015 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath in 819 msec
2024-11-17T20:35:13,803 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-17T20:35:13,803 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-11-17T20:35:13,805 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath
2024-11-17T20:35:13,806 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath Metrics about Tables on a single HBase RegionServer
2024-11-17T20:35:13,807 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-17T20:35:13,807 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-11-17T20:35:14,369 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-17T20:35:14,375 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:35:14,401 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:35:14,404 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:35:14,405 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:35:14,418 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnDatanodeDeath'
2024-11-17T20:35:19,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40559 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-17T20:35:19,218 INFO  [RPCClient-NioEventLoopGroup-4-11 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnDatanodeDeath completed
2024-11-17T20:35:19,218 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRollOnDatanodeDeath,, stopping at row=TestLogRolling-testLogRollOnDatanodeDeath ,, for max=2147483647 with caching=100
2024-11-17T20:35:19,221 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRollOnDatanodeDeath
2024-11-17T20:35:19,221 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRollOnDatanodeDeath,,1731875709192.6ea106502026e091c4a7f9fa49641279.
2024-11-17T20:35:19,238 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-17T20:35:19,241 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-17T20:35:19,242 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-17T20:35:19,242 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-17T20:35:19,242 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-17T20:35:19,243 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7c141b19{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/hadoop.log.dir/,AVAILABLE}
2024-11-17T20:35:19,243 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@25509568{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-17T20:35:19,351 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5b3e7853{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/java.io.tmpdir/jetty-localhost-34707-hadoop-hdfs-3_4_1-tests_jar-_-any-18049580779498844687/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T20:35:19,351 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@65429201{HTTP/1.1, (http/1.1)}{localhost:34707}
2024-11-17T20:35:19,351 INFO  [Time-limited test {}] server.Server(415): Started @122637ms
2024-11-17T20:35:19,353 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-17T20:35:19,382 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-17T20:35:19,387 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-17T20:35:19,387 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-17T20:35:19,388 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-17T20:35:19,388 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-17T20:35:19,388 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5cbf28ae{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/hadoop.log.dir/,AVAILABLE}
2024-11-17T20:35:19,389 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@412902c6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-17T20:35:19,494 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@30fb5696{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/java.io.tmpdir/jetty-localhost-43793-hadoop-hdfs-3_4_1-tests_jar-_-any-1715307684629004279/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T20:35:19,494 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@70a0068f{HTTP/1.1, (http/1.1)}{localhost:43793}
2024-11-17T20:35:19,494 INFO  [Time-limited test {}] server.Server(415): Started @122780ms
2024-11-17T20:35:19,495 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-17T20:35:19,526 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-17T20:35:19,530 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-17T20:35:19,530 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-17T20:35:19,530 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-17T20:35:19,530 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-17T20:35:19,531 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@92842f7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/hadoop.log.dir/,AVAILABLE}
2024-11-17T20:35:19,531 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7d285dba{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-17T20:35:19,656 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@442cf16f{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/java.io.tmpdir/jetty-localhost-36761-hadoop-hdfs-3_4_1-tests_jar-_-any-10726686616196715010/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T20:35:19,657 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5ec5aa68{HTTP/1.1, (http/1.1)}{localhost:36761}
2024-11-17T20:35:19,657 INFO  [Time-limited test {}] server.Server(415): Started @122943ms
2024-11-17T20:35:19,658 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-17T20:35:21,035 WARN  [Thread-867 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/cluster_b9ca40e8-4224-d8ab-7466-d1512b745e25/data/data5/current/BP-1477976244-172.17.0.2-1731875704920/current, will proceed with Du for space computation calculation, 
2024-11-17T20:35:21,035 WARN  [Thread-868 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/cluster_b9ca40e8-4224-d8ab-7466-d1512b745e25/data/data6/current/BP-1477976244-172.17.0.2-1731875704920/current, will proceed with Du for space computation calculation, 
2024-11-17T20:35:21,055 WARN  [Thread-808 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-17T20:35:21,057 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x97a1f455460fa660 with lease ID 0x9923bdc54ecc4194: Processing first storage report for DS-c88e28b9-4de5-4077-a5d2-a5f42ad2b57f from datanode DatanodeRegistration(127.0.0.1:38211, datanodeUuid=91f64229-4163-4c75-867e-a89f1c44de82, infoPort=46649, infoSecurePort=0, ipcPort=34669, storageInfo=lv=-57;cid=testClusterID;nsid=991965062;c=1731875704920)
2024-11-17T20:35:21,057 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x97a1f455460fa660 with lease ID 0x9923bdc54ecc4194: from storage DS-c88e28b9-4de5-4077-a5d2-a5f42ad2b57f node DatanodeRegistration(127.0.0.1:38211, datanodeUuid=91f64229-4163-4c75-867e-a89f1c44de82, infoPort=46649, infoSecurePort=0, ipcPort=34669, storageInfo=lv=-57;cid=testClusterID;nsid=991965062;c=1731875704920), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-17T20:35:21,058 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x97a1f455460fa660 with lease ID 0x9923bdc54ecc4194: Processing first storage report for DS-a6044053-c9e0-44e5-9e7c-570bc5d57f4e from datanode DatanodeRegistration(127.0.0.1:38211, datanodeUuid=91f64229-4163-4c75-867e-a89f1c44de82, infoPort=46649, infoSecurePort=0, ipcPort=34669, storageInfo=lv=-57;cid=testClusterID;nsid=991965062;c=1731875704920)
2024-11-17T20:35:21,058 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x97a1f455460fa660 with lease ID 0x9923bdc54ecc4194: from storage DS-a6044053-c9e0-44e5-9e7c-570bc5d57f4e node DatanodeRegistration(127.0.0.1:38211, datanodeUuid=91f64229-4163-4c75-867e-a89f1c44de82, infoPort=46649, infoSecurePort=0, ipcPort=34669, storageInfo=lv=-57;cid=testClusterID;nsid=991965062;c=1731875704920), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-17T20:35:21,343 WARN  [Thread-878 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/cluster_b9ca40e8-4224-d8ab-7466-d1512b745e25/data/data7/current/BP-1477976244-172.17.0.2-1731875704920/current, will proceed with Du for space computation calculation, 
2024-11-17T20:35:21,343 WARN  [Thread-879 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/cluster_b9ca40e8-4224-d8ab-7466-d1512b745e25/data/data8/current/BP-1477976244-172.17.0.2-1731875704920/current, will proceed with Du for space computation calculation, 
2024-11-17T20:35:21,363 WARN  [Thread-830 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-17T20:35:21,366 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1c40e94a2e4bb6aa with lease ID 0x9923bdc54ecc4195: Processing first storage report for DS-71cd9595-34b3-4e78-8dba-8b432c0955e2 from datanode DatanodeRegistration(127.0.0.1:35683, datanodeUuid=b56e73c4-1141-4ee7-8e8e-d9751264448b, infoPort=39349, infoSecurePort=0, ipcPort=35057, storageInfo=lv=-57;cid=testClusterID;nsid=991965062;c=1731875704920)
2024-11-17T20:35:21,366 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1c40e94a2e4bb6aa with lease ID 0x9923bdc54ecc4195: from storage DS-71cd9595-34b3-4e78-8dba-8b432c0955e2 node DatanodeRegistration(127.0.0.1:35683, datanodeUuid=b56e73c4-1141-4ee7-8e8e-d9751264448b, infoPort=39349, infoSecurePort=0, ipcPort=35057, storageInfo=lv=-57;cid=testClusterID;nsid=991965062;c=1731875704920), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-17T20:35:21,366 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1c40e94a2e4bb6aa with lease ID 0x9923bdc54ecc4195: Processing first storage report for DS-acd1cddb-0eca-4d32-ba22-8d6f2c3add4f from datanode DatanodeRegistration(127.0.0.1:35683, datanodeUuid=b56e73c4-1141-4ee7-8e8e-d9751264448b, infoPort=39349, infoSecurePort=0, ipcPort=35057, storageInfo=lv=-57;cid=testClusterID;nsid=991965062;c=1731875704920)
2024-11-17T20:35:21,366 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1c40e94a2e4bb6aa with lease ID 0x9923bdc54ecc4195: from storage DS-acd1cddb-0eca-4d32-ba22-8d6f2c3add4f node DatanodeRegistration(127.0.0.1:35683, datanodeUuid=b56e73c4-1141-4ee7-8e8e-d9751264448b, infoPort=39349, infoSecurePort=0, ipcPort=35057, storageInfo=lv=-57;cid=testClusterID;nsid=991965062;c=1731875704920), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-17T20:35:21,396 WARN  [Thread-889 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/cluster_b9ca40e8-4224-d8ab-7466-d1512b745e25/data/data9/current/BP-1477976244-172.17.0.2-1731875704920/current, will proceed with Du for space computation calculation, 
2024-11-17T20:35:21,396 WARN  [Thread-890 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/cluster_b9ca40e8-4224-d8ab-7466-d1512b745e25/data/data10/current/BP-1477976244-172.17.0.2-1731875704920/current, will proceed with Du for space computation calculation, 
2024-11-17T20:35:21,417 WARN  [Thread-852 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-17T20:35:21,420 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf80d8f639c184837 with lease ID 0x9923bdc54ecc4196: Processing first storage report for DS-e3ca24df-f174-47d5-ac27-0b38fd48e050 from datanode DatanodeRegistration(127.0.0.1:35007, datanodeUuid=96f27179-3395-467d-8cec-ee4e6e5e306e, infoPort=34837, infoSecurePort=0, ipcPort=46245, storageInfo=lv=-57;cid=testClusterID;nsid=991965062;c=1731875704920)
2024-11-17T20:35:21,420 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf80d8f639c184837 with lease ID 0x9923bdc54ecc4196: from storage DS-e3ca24df-f174-47d5-ac27-0b38fd48e050 node DatanodeRegistration(127.0.0.1:35007, datanodeUuid=96f27179-3395-467d-8cec-ee4e6e5e306e, infoPort=34837, infoSecurePort=0, ipcPort=46245, storageInfo=lv=-57;cid=testClusterID;nsid=991965062;c=1731875704920), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-17T20:35:21,420 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf80d8f639c184837 with lease ID 0x9923bdc54ecc4196: Processing first storage report for DS-0d8c7247-7b53-4633-babc-2bf53fc3b966 from datanode DatanodeRegistration(127.0.0.1:35007, datanodeUuid=96f27179-3395-467d-8cec-ee4e6e5e306e, infoPort=34837, infoSecurePort=0, ipcPort=46245, storageInfo=lv=-57;cid=testClusterID;nsid=991965062;c=1731875704920)
2024-11-17T20:35:21,420 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf80d8f639c184837 with lease ID 0x9923bdc54ecc4196: from storage DS-0d8c7247-7b53-4633-babc-2bf53fc3b966 node DatanodeRegistration(127.0.0.1:35007, datanodeUuid=96f27179-3395-467d-8cec-ee4e6e5e306e, infoPort=34837, infoSecurePort=0, ipcPort=46245, storageInfo=lv=-57;cid=testClusterID;nsid=991965062;c=1731875704920), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-17T20:35:21,495 WARN  [ResponseProcessor for block BP-1477976244-172.17.0.2-1731875704920:blk_1073741832_1008 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1477976244-172.17.0.2-1731875704920:blk_1073741832_1008
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:21,495 WARN  [ResponseProcessor for block BP-1477976244-172.17.0.2-1731875704920:blk_1073741836_1012 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1477976244-172.17.0.2-1731875704920:blk_1073741836_1012
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:21,496 WARN  [ResponseProcessor for block BP-1477976244-172.17.0.2-1731875704920:blk_1073741830_1006 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1477976244-172.17.0.2-1731875704920:blk_1073741830_1006
java.io.IOException: Bad response ERROR for BP-1477976244-172.17.0.2-1731875704920:blk_1073741830_1006 from datanode DatanodeInfoWithStorage[127.0.0.1:42609,DS-59ce2296-6caf-48fd-a174-2d36e11610d2,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:21,496 WARN  [DataStreamer for file /user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295 block BP-1477976244-172.17.0.2-1731875704920:blk_1073741836_1012 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1477976244-172.17.0.2-1731875704920:blk_1073741836_1012 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42609,DS-59ce2296-6caf-48fd-a174-2d36e11610d2,DISK], DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42609,DS-59ce2296-6caf-48fd-a174-2d36e11610d2,DISK]) is bad.
2024-11-17T20:35:21,497 WARN  [DataStreamer for file /user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650 block BP-1477976244-172.17.0.2-1731875704920:blk_1073741830_1006 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1477976244-172.17.0.2-1731875704920:blk_1073741830_1006 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK], DatanodeInfoWithStorage[127.0.0.1:42609,DS-59ce2296-6caf-48fd-a174-2d36e11610d2,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:42609,DS-59ce2296-6caf-48fd-a174-2d36e11610d2,DISK]) is bad.
2024-11-17T20:35:21,498 WARN  [PacketResponder: BP-1477976244-172.17.0.2-1731875704920:blk_1073741830_1006, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:42609] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:35:21,498 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_989359345_22 at /127.0.0.1:58296 [Receiving block BP-1477976244-172.17.0.2-1731875704920:blk_1073741836_1012] {}] datanode.DataXceiver(331): 127.0.0.1:42609:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:58296 dst: /127.0.0.1:42609
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:35:21,499 WARN  [ResponseProcessor for block BP-1477976244-172.17.0.2-1731875704920:blk_1073741834_1010 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1477976244-172.17.0.2-1731875704920:blk_1073741834_1010
java.io.IOException: Bad response ERROR for BP-1477976244-172.17.0.2-1731875704920:blk_1073741834_1010 from datanode DatanodeInfoWithStorage[127.0.0.1:42609,DS-59ce2296-6caf-48fd-a174-2d36e11610d2,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:21,500 WARN  [DataStreamer for file /user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta block BP-1477976244-172.17.0.2-1731875704920:blk_1073741834_1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1477976244-172.17.0.2-1731875704920:blk_1073741834_1010 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK], DatanodeInfoWithStorage[127.0.0.1:42609,DS-59ce2296-6caf-48fd-a174-2d36e11610d2,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:42609,DS-59ce2296-6caf-48fd-a174-2d36e11610d2,DISK]) is bad.
2024-11-17T20:35:21,500 WARN  [PacketResponder: BP-1477976244-172.17.0.2-1731875704920:blk_1073741834_1010, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:42609] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:35:21,501 WARN  [DataStreamer for file /user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.1731875708280 block BP-1477976244-172.17.0.2-1731875704920:blk_1073741832_1008 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1477976244-172.17.0.2-1731875704920:blk_1073741832_1008 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42609,DS-59ce2296-6caf-48fd-a174-2d36e11610d2,DISK], DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42609,DS-59ce2296-6caf-48fd-a174-2d36e11610d2,DISK]) is bad.
2024-11-17T20:35:21,502 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-723733033_22 at /127.0.0.1:34656 [Receiving block BP-1477976244-172.17.0.2-1731875704920:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:41145:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:34656 dst: /127.0.0.1:41145
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:35:21,501 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-723733033_22 at /127.0.0.1:58290 [Receiving block BP-1477976244-172.17.0.2-1731875704920:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:42609:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:58290 dst: /127.0.0.1:42609
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:35:21,502 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-723733033_22 at /127.0.0.1:58274 [Receiving block BP-1477976244-172.17.0.2-1731875704920:blk_1073741832_1008] {}] datanode.DataXceiver(331): 127.0.0.1:42609:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:58274 dst: /127.0.0.1:42609
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:35:21,502 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1228150533_22 at /127.0.0.1:34604 [Receiving block BP-1477976244-172.17.0.2-1731875704920:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:41145:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:34604 dst: /127.0.0.1:41145
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:35:21,502 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-723733033_22 at /127.0.0.1:34628 [Receiving block BP-1477976244-172.17.0.2-1731875704920:blk_1073741832_1008] {}] datanode.DataXceiver(331): 127.0.0.1:41145:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:34628 dst: /127.0.0.1:41145
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:35:21,502 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1228150533_22 at /127.0.0.1:58250 [Receiving block BP-1477976244-172.17.0.2-1731875704920:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:42609:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:58250 dst: /127.0.0.1:42609
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:35:21,506 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5a63af9{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T20:35:21,506 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3efce601{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-17T20:35:21,506 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-17T20:35:21,507 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@45b2e9f0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-17T20:35:21,507 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@461a42e8{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/hadoop.log.dir/,STOPPED}
2024-11-17T20:35:21,508 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_989359345_22 at /127.0.0.1:34674 [Receiving block BP-1477976244-172.17.0.2-1731875704920:blk_1073741836_1012] {}] datanode.DataXceiver(331): 127.0.0.1:41145:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:34674 dst: /127.0.0.1:41145
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:35:21,509 WARN  [BP-1477976244-172.17.0.2-1731875704920 heartbeating to localhost/127.0.0.1:33551 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-17T20:35:21,509 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-17T20:35:21,509 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-17T20:35:21,509 WARN  [BP-1477976244-172.17.0.2-1731875704920 heartbeating to localhost/127.0.0.1:33551 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1477976244-172.17.0.2-1731875704920 (Datanode Uuid 6506fff5-03d5-479e-8b59-d4ac07be411d) service to localhost/127.0.0.1:33551
2024-11-17T20:35:21,510 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/cluster_b9ca40e8-4224-d8ab-7466-d1512b745e25/data/data3/current/BP-1477976244-172.17.0.2-1731875704920 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T20:35:21,510 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/cluster_b9ca40e8-4224-d8ab-7466-d1512b745e25/data/data4/current/BP-1477976244-172.17.0.2-1731875704920 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T20:35:21,510 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-17T20:35:21,511 WARN  [DataStreamer for file /user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295 block BP-1477976244-172.17.0.2-1731875704920:blk_1073741836_1012 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741836_1012
java.net.SocketException: Connection reset
	at sun.nio.ch.SocketChannelImpl.throwConnectionReset(SocketChannelImpl.java:394) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:426) ~[?:?]
	at org.apache.hadoop.net.SocketInputStream$Reader.performIO(SocketInputStream.java:57) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:118) ~[hadoop-common-3.4.1.jar:?]
	at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?]
	at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:527) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1931) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:21,512 WARN  [DataStreamer for file /user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta block BP-1477976244-172.17.0.2-1731875704920:blk_1073741834_1010 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741834_1010
java.net.SocketException: Connection reset
	at sun.nio.ch.SocketChannelImpl.throwConnectionReset(SocketChannelImpl.java:394) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:426) ~[?:?]
	at org.apache.hadoop.net.SocketInputStream$Reader.performIO(SocketInputStream.java:57) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:118) ~[hadoop-common-3.4.1.jar:?]
	at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?]
	at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:527) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1931) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:21,512 ERROR [org.apache.hadoop.hdfs.server.datanode.DataXceiver@555aedc9 {}] datanode.DataXceiver(331): 127.0.0.1:41145:DataXceiver error processing unknown operation  src: /127.0.0.1:40836 dst: /127.0.0.1:41145
java.io.IOException: Server closed.
	at org.apache.hadoop.hdfs.server.datanode.DataXceiverServer.addPeer(DataXceiverServer.java:334) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:232) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:35:21,513 WARN  [DataStreamer for file /user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.1731875708280 block BP-1477976244-172.17.0.2-1731875704920:blk_1073741832_1008 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741832_1008
java.net.SocketException: Connection reset
	at sun.nio.ch.SocketChannelImpl.throwConnectionReset(SocketChannelImpl.java:394) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:426) ~[?:?]
	at org.apache.hadoop.net.SocketInputStream$Reader.performIO(SocketInputStream.java:57) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:118) ~[hadoop-common-3.4.1.jar:?]
	at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?]
	at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:527) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1931) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:21,513 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1228150533_22 at /127.0.0.1:40838 [Receiving block BP-1477976244-172.17.0.2-1731875704920:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:41145:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:40838 dst: /127.0.0.1:41145
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:35:21,513 WARN  [ResponseProcessor for block BP-1477976244-172.17.0.2-1731875704920:blk_1073741830_1014 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1477976244-172.17.0.2-1731875704920:blk_1073741830_1014
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:21,515 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@68605b4b{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T20:35:21,516 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@158b5ab3{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-17T20:35:21,516 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-17T20:35:21,516 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@805be33{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-17T20:35:21,517 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4577488a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/hadoop.log.dir/,STOPPED}
2024-11-17T20:35:21,518 WARN  [BP-1477976244-172.17.0.2-1731875704920 heartbeating to localhost/127.0.0.1:33551 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-17T20:35:21,518 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-17T20:35:21,518 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-17T20:35:21,518 WARN  [BP-1477976244-172.17.0.2-1731875704920 heartbeating to localhost/127.0.0.1:33551 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1477976244-172.17.0.2-1731875704920 (Datanode Uuid 15f63053-16f8-4ae9-8ca1-f284e5d8b564) service to localhost/127.0.0.1:33551
2024-11-17T20:35:21,518 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/cluster_b9ca40e8-4224-d8ab-7466-d1512b745e25/data/data1/current/BP-1477976244-172.17.0.2-1731875704920 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T20:35:21,519 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/cluster_b9ca40e8-4224-d8ab-7466-d1512b745e25/data/data2/current/BP-1477976244-172.17.0.2-1731875704920 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T20:35:21,519 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-17T20:35:21,522 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRollOnDatanodeDeath', row='row0002', locateType=CURRENT is [region=TestLogRolling-testLogRollOnDatanodeDeath,,1731875709192.6ea106502026e091c4a7f9fa49641279., hostname=0b0b0ae8de03,33133,1731875707434, seqNum=2]
2024-11-17T20:35:21,524 ERROR [FSHLog-0-hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899-prefix:0b0b0ae8de03,33133,1731875707434 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:21,524 WARN  [FSHLog-0-hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899-prefix:0b0b0ae8de03,33133,1731875707434 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:21,525 INFO  [regionserver/0b0b0ae8de03:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:21,525 DEBUG [regionserver/0b0b0ae8de03:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 0b0b0ae8de03%2C33133%2C1731875707434:(num 1731875708280) roll requested
2024-11-17T20:35:21,525 INFO  [regionserver/0b0b0ae8de03:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0b0b0ae8de03%2C33133%2C1731875707434.1731875721525
2024-11-17T20:35:21,528 WARN  [Thread-904 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741838_1018
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:21,528 WARN  [Thread-904 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1477976244-172.17.0.2-1731875704920:blk_1073741838_1018 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42609,DS-59ce2296-6caf-48fd-a174-2d36e11610d2,DISK], DatanodeInfoWithStorage[127.0.0.1:38211,DS-c88e28b9-4de5-4077-a5d2-a5f42ad2b57f,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42609,DS-59ce2296-6caf-48fd-a174-2d36e11610d2,DISK]) is bad.
2024-11-17T20:35:21,529 WARN  [Thread-904 {}] hdfs.DataStreamer(1850): Abandoning BP-1477976244-172.17.0.2-1731875704920:blk_1073741838_1018
2024-11-17T20:35:21,531 WARN  [Thread-904 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42609,DS-59ce2296-6caf-48fd-a174-2d36e11610d2,DISK]
2024-11-17T20:35:21,537 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:35:21,538 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:35:21,538 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:35:21,538 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:35:21,538 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:35:21,538 INFO  [regionserver/0b0b0ae8de03:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.1731875708280 with entries=1, filesize=455 B; new WAL /user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.1731875721525
2024-11-17T20:35:21,538 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:21,539 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:21,540 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils
2024-11-17T20:35:21,540 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease()
2024-11-17T20:35:21,540 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.1731875708280
2024-11-17T20:35:21,540 DEBUG [regionserver/0b0b0ae8de03:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46649:46649),(127.0.0.1/127.0.0.1:39349:39349)]
2024-11-17T20:35:21,540 DEBUG [regionserver/0b0b0ae8de03:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.1731875708280 is not closed yet, will try archiving it next time
2024-11-17T20:35:21,543 WARN  [IPC Server handler 0 on default port 33551 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.1731875708280 has not been closed. Lease recovery is in progress. RecoveryId = 1020 for block blk_1073741832_1008
2024-11-17T20:35:21,546 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.1731875708280 after 5ms
2024-11-17T20:35:21,617 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:23,152 INFO  [regionserver/0b0b0ae8de03:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:23,541 INFO  [regionserver/0b0b0ae8de03:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:23,542 INFO  [Time-limited test {}] wal.TestLogRolling(261): log.getCurrentFileName(): hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.1731875721525
2024-11-17T20:35:23,543 WARN  [ResponseProcessor for block BP-1477976244-172.17.0.2-1731875704920:blk_1073741839_1019 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1477976244-172.17.0.2-1731875704920:blk_1073741839_1019
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:23,543 WARN  [DataStreamer for file /user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.1731875721525 block BP-1477976244-172.17.0.2-1731875704920:blk_1073741839_1019 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1477976244-172.17.0.2-1731875704920:blk_1073741839_1019 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38211,DS-c88e28b9-4de5-4077-a5d2-a5f42ad2b57f,DISK], DatanodeInfoWithStorage[127.0.0.1:35683,DS-71cd9595-34b3-4e78-8dba-8b432c0955e2,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38211,DS-c88e28b9-4de5-4077-a5d2-a5f42ad2b57f,DISK]) is bad.
2024-11-17T20:35:23,543 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-723733033_22 at /127.0.0.1:60382 [Receiving block BP-1477976244-172.17.0.2-1731875704920:blk_1073741839_1019] {}] datanode.DataXceiver(331): 127.0.0.1:38211:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:60382 dst: /127.0.0.1:38211
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:35:23,544 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-723733033_22 at /127.0.0.1:37498 [Receiving block BP-1477976244-172.17.0.2-1731875704920:blk_1073741839_1019] {}] datanode.DataXceiver(331): 127.0.0.1:35683:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:37498 dst: /127.0.0.1:35683
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:35:23,578 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5b3e7853{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T20:35:23,579 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@65429201{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-17T20:35:23,579 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-17T20:35:23,579 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@25509568{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-17T20:35:23,579 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7c141b19{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/hadoop.log.dir/,STOPPED}
2024-11-17T20:35:23,581 WARN  [BP-1477976244-172.17.0.2-1731875704920 heartbeating to localhost/127.0.0.1:33551 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-17T20:35:23,581 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-17T20:35:23,581 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-17T20:35:23,581 WARN  [BP-1477976244-172.17.0.2-1731875704920 heartbeating to localhost/127.0.0.1:33551 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1477976244-172.17.0.2-1731875704920 (Datanode Uuid 91f64229-4163-4c75-867e-a89f1c44de82) service to localhost/127.0.0.1:33551
2024-11-17T20:35:23,582 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/cluster_b9ca40e8-4224-d8ab-7466-d1512b745e25/data/data5/current/BP-1477976244-172.17.0.2-1731875704920 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T20:35:23,582 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/cluster_b9ca40e8-4224-d8ab-7466-d1512b745e25/data/data6/current/BP-1477976244-172.17.0.2-1731875704920 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T20:35:23,582 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-17T20:35:23,618 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:25,152 INFO  [regionserver/0b0b0ae8de03:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:25,541 WARN  [regionserver/0b0b0ae8de03:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:35683,DS-71cd9595-34b3-4e78-8dba-8b432c0955e2,DISK]]
2024-11-17T20:35:25,542 INFO  [regionserver/0b0b0ae8de03:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:25,542 DEBUG [regionserver/0b0b0ae8de03:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 0b0b0ae8de03%2C33133%2C1731875707434:(num 1731875721525) roll requested
2024-11-17T20:35:25,542 INFO  [regionserver/0b0b0ae8de03:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0b0b0ae8de03%2C33133%2C1731875707434.1731875725542
2024-11-17T20:35:25,548 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.1731875708280 after 4008ms
2024-11-17T20:35:25,548 WARN  [Thread-913 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741840_1022
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:42609
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:25,548 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-723733033_22 at /127.0.0.1:47246 [Receiving block BP-1477976244-172.17.0.2-1731875704920:blk_1073741840_1022] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/cluster_b9ca40e8-4224-d8ab-7466-d1512b745e25/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/cluster_b9ca40e8-4224-d8ab-7466-d1512b745e25/data/data10]'}, localName='127.0.0.1:35007', datanodeUuid='96f27179-3395-467d-8cec-ee4e6e5e306e', xmitsInProgress=0}:Exception transferring block BP-1477976244-172.17.0.2-1731875704920:blk_1073741840_1022 to mirror 127.0.0.1:42609
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:35:25,549 WARN  [Thread-913 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1477976244-172.17.0.2-1731875704920:blk_1073741840_1022 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35007,DS-e3ca24df-f174-47d5-ac27-0b38fd48e050,DISK], DatanodeInfoWithStorage[127.0.0.1:42609,DS-59ce2296-6caf-48fd-a174-2d36e11610d2,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:42609,DS-59ce2296-6caf-48fd-a174-2d36e11610d2,DISK]) is bad.
2024-11-17T20:35:25,549 WARN  [Thread-913 {}] hdfs.DataStreamer(1850): Abandoning BP-1477976244-172.17.0.2-1731875704920:blk_1073741840_1022
2024-11-17T20:35:25,549 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-723733033_22 at /127.0.0.1:47246 [Receiving block BP-1477976244-172.17.0.2-1731875704920:blk_1073741840_1022] {}] datanode.BlockReceiver(316): Block 1073741840 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-17T20:35:25,549 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-723733033_22 at /127.0.0.1:47246 [Receiving block BP-1477976244-172.17.0.2-1731875704920:blk_1073741840_1022] {}] datanode.DataXceiver(331): 127.0.0.1:35007:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:47246 dst: /127.0.0.1:35007
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:35:25,550 WARN  [Thread-913 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42609,DS-59ce2296-6caf-48fd-a174-2d36e11610d2,DISK]
2024-11-17T20:35:25,553 WARN  [Thread-913 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741841_1023
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:41145
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:25,553 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-723733033_22 at /127.0.0.1:33838 [Receiving block BP-1477976244-172.17.0.2-1731875704920:blk_1073741841_1023] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/cluster_b9ca40e8-4224-d8ab-7466-d1512b745e25/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/cluster_b9ca40e8-4224-d8ab-7466-d1512b745e25/data/data8]'}, localName='127.0.0.1:35683', datanodeUuid='b56e73c4-1141-4ee7-8e8e-d9751264448b', xmitsInProgress=0}:Exception transferring block BP-1477976244-172.17.0.2-1731875704920:blk_1073741841_1023 to mirror 127.0.0.1:41145
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:35:25,553 WARN  [Thread-913 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1477976244-172.17.0.2-1731875704920:blk_1073741841_1023 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35683,DS-71cd9595-34b3-4e78-8dba-8b432c0955e2,DISK], DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]) is bad.
2024-11-17T20:35:25,553 WARN  [Thread-913 {}] hdfs.DataStreamer(1850): Abandoning BP-1477976244-172.17.0.2-1731875704920:blk_1073741841_1023
2024-11-17T20:35:25,553 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-723733033_22 at /127.0.0.1:33838 [Receiving block BP-1477976244-172.17.0.2-1731875704920:blk_1073741841_1023] {}] datanode.BlockReceiver(316): Block 1073741841 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-17T20:35:25,554 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-723733033_22 at /127.0.0.1:33838 [Receiving block BP-1477976244-172.17.0.2-1731875704920:blk_1073741841_1023] {}] datanode.DataXceiver(331): 127.0.0.1:35683:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:33838 dst: /127.0.0.1:35683
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:35:25,554 WARN  [Thread-913 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]
2024-11-17T20:35:25,556 WARN  [Thread-913 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741842_1024
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:25,556 WARN  [Thread-913 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1477976244-172.17.0.2-1731875704920:blk_1073741842_1024 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38211,DS-c88e28b9-4de5-4077-a5d2-a5f42ad2b57f,DISK], DatanodeInfoWithStorage[127.0.0.1:35007,DS-e3ca24df-f174-47d5-ac27-0b38fd48e050,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38211,DS-c88e28b9-4de5-4077-a5d2-a5f42ad2b57f,DISK]) is bad.
2024-11-17T20:35:25,556 WARN  [Thread-913 {}] hdfs.DataStreamer(1850): Abandoning BP-1477976244-172.17.0.2-1731875704920:blk_1073741842_1024
2024-11-17T20:35:25,557 WARN  [Thread-913 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38211,DS-c88e28b9-4de5-4077-a5d2-a5f42ad2b57f,DISK]
2024-11-17T20:35:25,562 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:35:25,562 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:35:25,562 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:35:25,562 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:35:25,562 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:35:25,562 INFO  [regionserver/0b0b0ae8de03:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.1731875721525 with entries=2, filesize=2.37 KB; new WAL /user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.1731875725542
2024-11-17T20:35:25,563 DEBUG [regionserver/0b0b0ae8de03:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34837:34837),(127.0.0.1/127.0.0.1:39349:39349)]
2024-11-17T20:35:25,563 DEBUG [regionserver/0b0b0ae8de03:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.1731875708280 is not closed yet, will try archiving it next time
2024-11-17T20:35:25,563 DEBUG [regionserver/0b0b0ae8de03:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.1731875721525 is not closed yet, will try archiving it next time
2024-11-17T20:35:25,564 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35683 is added to blk_1073741839_1021 (size=2431)
2024-11-17T20:35:25,588 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2024-11-17T20:35:25,618 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:25,965 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.1731875708280 is not closed yet, will try archiving it next time
2024-11-17T20:35:27,153 INFO  [regionserver/0b0b0ae8de03:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:27,383 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@325a1a43[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:35683, datanodeUuid=b56e73c4-1141-4ee7-8e8e-d9751264448b, infoPort=39349, infoSecurePort=0, ipcPort=35057, storageInfo=lv=-57;cid=testClusterID;nsid=991965062;c=1731875704920):Failed to transfer BP-1477976244-172.17.0.2-1731875704920:blk_1073741839_1021 to 127.0.0.1:38211 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:35:27,563 INFO  [regionserver/0b0b0ae8de03:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:27,593 WARN  [ResponseProcessor for block BP-1477976244-172.17.0.2-1731875704920:blk_1073741843_1025 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1477976244-172.17.0.2-1731875704920:blk_1073741843_1025
java.io.IOException: Bad response ERROR for BP-1477976244-172.17.0.2-1731875704920:blk_1073741843_1025 from datanode DatanodeInfoWithStorage[127.0.0.1:35683,DS-71cd9595-34b3-4e78-8dba-8b432c0955e2,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:27,593 WARN  [DataStreamer for file /user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.1731875725542 block BP-1477976244-172.17.0.2-1731875704920:blk_1073741843_1025 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1477976244-172.17.0.2-1731875704920:blk_1073741843_1025 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35007,DS-e3ca24df-f174-47d5-ac27-0b38fd48e050,DISK], DatanodeInfoWithStorage[127.0.0.1:35683,DS-71cd9595-34b3-4e78-8dba-8b432c0955e2,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:35683,DS-71cd9595-34b3-4e78-8dba-8b432c0955e2,DISK]) is bad.
2024-11-17T20:35:27,593 WARN  [PacketResponder: BP-1477976244-172.17.0.2-1731875704920:blk_1073741843_1025, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:35683] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:35:27,594 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-723733033_22 at /127.0.0.1:47258 [Receiving block BP-1477976244-172.17.0.2-1731875704920:blk_1073741843_1025] {}] datanode.DataXceiver(331): 127.0.0.1:35007:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:47258 dst: /127.0.0.1:35007
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:35:27,594 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-723733033_22 at /127.0.0.1:33844 [Receiving block BP-1477976244-172.17.0.2-1731875704920:blk_1073741843_1025] {}] datanode.DataXceiver(331): 127.0.0.1:35683:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:33844 dst: /127.0.0.1:35683
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:35:27,619 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:27,629 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@30fb5696{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T20:35:27,630 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@70a0068f{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-17T20:35:27,630 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-17T20:35:27,630 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@412902c6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-17T20:35:27,630 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5cbf28ae{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/hadoop.log.dir/,STOPPED}
2024-11-17T20:35:27,632 WARN  [BP-1477976244-172.17.0.2-1731875704920 heartbeating to localhost/127.0.0.1:33551 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-17T20:35:27,632 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-17T20:35:27,632 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-17T20:35:27,632 WARN  [BP-1477976244-172.17.0.2-1731875704920 heartbeating to localhost/127.0.0.1:33551 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1477976244-172.17.0.2-1731875704920 (Datanode Uuid b56e73c4-1141-4ee7-8e8e-d9751264448b) service to localhost/127.0.0.1:33551
2024-11-17T20:35:27,633 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/cluster_b9ca40e8-4224-d8ab-7466-d1512b745e25/data/data7/current/BP-1477976244-172.17.0.2-1731875704920 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T20:35:27,633 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/cluster_b9ca40e8-4224-d8ab-7466-d1512b745e25/data/data8/current/BP-1477976244-172.17.0.2-1731875704920 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T20:35:27,633 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-17T20:35:27,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33133 {}] regionserver.HRegion(8855): Flush requested on 6ea106502026e091c4a7f9fa49641279
2024-11-17T20:35:27,643 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ea106502026e091c4a7f9fa49641279 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-17T20:35:27,664 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/.tmp/info/3f1fe0c9aab24b899b5dfe69f3f5857c is 1080, key is row0002/info:/1731875723584/Put/seqid=0
2024-11-17T20:35:27,666 WARN  [Thread-927 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741844_1027
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:27,666 WARN  [Thread-927 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1477976244-172.17.0.2-1731875704920:blk_1073741844_1027 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK], DatanodeInfoWithStorage[127.0.0.1:35007,DS-e3ca24df-f174-47d5-ac27-0b38fd48e050,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]) is bad.
2024-11-17T20:35:27,666 WARN  [Thread-927 {}] hdfs.DataStreamer(1850): Abandoning BP-1477976244-172.17.0.2-1731875704920:blk_1073741844_1027
2024-11-17T20:35:27,667 WARN  [Thread-927 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]
2024-11-17T20:35:27,668 WARN  [Thread-927 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741845_1028
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:27,668 WARN  [Thread-927 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1477976244-172.17.0.2-1731875704920:blk_1073741845_1028 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35683,DS-71cd9595-34b3-4e78-8dba-8b432c0955e2,DISK], DatanodeInfoWithStorage[127.0.0.1:35007,DS-e3ca24df-f174-47d5-ac27-0b38fd48e050,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35683,DS-71cd9595-34b3-4e78-8dba-8b432c0955e2,DISK]) is bad.
2024-11-17T20:35:27,668 WARN  [Thread-927 {}] hdfs.DataStreamer(1850): Abandoning BP-1477976244-172.17.0.2-1731875704920:blk_1073741845_1028
2024-11-17T20:35:27,669 WARN  [Thread-927 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35683,DS-71cd9595-34b3-4e78-8dba-8b432c0955e2,DISK]
2024-11-17T20:35:27,670 WARN  [Thread-927 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741846_1029
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:27,671 WARN  [Thread-927 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1477976244-172.17.0.2-1731875704920:blk_1073741846_1029 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42609,DS-59ce2296-6caf-48fd-a174-2d36e11610d2,DISK], DatanodeInfoWithStorage[127.0.0.1:35007,DS-e3ca24df-f174-47d5-ac27-0b38fd48e050,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42609,DS-59ce2296-6caf-48fd-a174-2d36e11610d2,DISK]) is bad.
2024-11-17T20:35:27,671 WARN  [Thread-927 {}] hdfs.DataStreamer(1850): Abandoning BP-1477976244-172.17.0.2-1731875704920:blk_1073741846_1029
2024-11-17T20:35:27,671 WARN  [Thread-927 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42609,DS-59ce2296-6caf-48fd-a174-2d36e11610d2,DISK]
2024-11-17T20:35:27,673 WARN  [Thread-927 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741847_1030
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:27,673 WARN  [Thread-927 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1477976244-172.17.0.2-1731875704920:blk_1073741847_1030 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38211,DS-c88e28b9-4de5-4077-a5d2-a5f42ad2b57f,DISK], DatanodeInfoWithStorage[127.0.0.1:35007,DS-e3ca24df-f174-47d5-ac27-0b38fd48e050,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38211,DS-c88e28b9-4de5-4077-a5d2-a5f42ad2b57f,DISK]) is bad.
2024-11-17T20:35:27,673 WARN  [Thread-927 {}] hdfs.DataStreamer(1850): Abandoning BP-1477976244-172.17.0.2-1731875704920:blk_1073741847_1030
2024-11-17T20:35:27,673 WARN  [Thread-927 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38211,DS-c88e28b9-4de5-4077-a5d2-a5f42ad2b57f,DISK]
2024-11-17T20:35:27,674 WARN  [IPC Server handler 0 on default port 33551 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-17T20:35:27,674 WARN  [IPC Server handler 0 on default port 33551 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-17T20:35:27,674 WARN  [IPC Server handler 0 on default port 33551 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-17T20:35:27,677 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35007 is added to blk_1073741848_1031 (size=10347)
2024-11-17T20:35:28,078 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/.tmp/info/3f1fe0c9aab24b899b5dfe69f3f5857c
2024-11-17T20:35:28,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/.tmp/info/3f1fe0c9aab24b899b5dfe69f3f5857c as hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/info/3f1fe0c9aab24b899b5dfe69f3f5857c
2024-11-17T20:35:28,096 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/info/3f1fe0c9aab24b899b5dfe69f3f5857c, entries=5, sequenceid=11, filesize=10.1 K
2024-11-17T20:35:28,098 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=9.45 KB/9681 for 6ea106502026e091c4a7f9fa49641279 in 454ms, sequenceid=11, compaction requested=false
2024-11-17T20:35:28,098 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ea106502026e091c4a7f9fa49641279:

2024-11-17T20:35:28,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33133 {}] regionserver.HRegion(8855): Flush requested on 6ea106502026e091c4a7f9fa49641279
2024-11-17T20:35:28,273 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ea106502026e091c4a7f9fa49641279 1/1 column families, dataSize=10.50 KB heapSize=11.50 KB
2024-11-17T20:35:28,278 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/.tmp/info/a633fbb8c8994446b4e51b16e3c0a02d is 1080, key is row0007/info:/1731875727644/Put/seqid=0
2024-11-17T20:35:28,282 WARN  [Thread-932 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741849_1032
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:35683
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:28,282 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-723733033_22 at /127.0.0.1:47296 [Receiving block BP-1477976244-172.17.0.2-1731875704920:blk_1073741849_1032] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/cluster_b9ca40e8-4224-d8ab-7466-d1512b745e25/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/cluster_b9ca40e8-4224-d8ab-7466-d1512b745e25/data/data10]'}, localName='127.0.0.1:35007', datanodeUuid='96f27179-3395-467d-8cec-ee4e6e5e306e', xmitsInProgress=0}:Exception transferring block BP-1477976244-172.17.0.2-1731875704920:blk_1073741849_1032 to mirror 127.0.0.1:35683
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:35:28,282 WARN  [Thread-932 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1477976244-172.17.0.2-1731875704920:blk_1073741849_1032 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35007,DS-e3ca24df-f174-47d5-ac27-0b38fd48e050,DISK], DatanodeInfoWithStorage[127.0.0.1:35683,DS-71cd9595-34b3-4e78-8dba-8b432c0955e2,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:35683,DS-71cd9595-34b3-4e78-8dba-8b432c0955e2,DISK]) is bad.
2024-11-17T20:35:28,282 WARN  [Thread-932 {}] hdfs.DataStreamer(1850): Abandoning BP-1477976244-172.17.0.2-1731875704920:blk_1073741849_1032
2024-11-17T20:35:28,282 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-723733033_22 at /127.0.0.1:47296 [Receiving block BP-1477976244-172.17.0.2-1731875704920:blk_1073741849_1032] {}] datanode.BlockReceiver(316): Block 1073741849 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-17T20:35:28,282 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-723733033_22 at /127.0.0.1:47296 [Receiving block BP-1477976244-172.17.0.2-1731875704920:blk_1073741849_1032] {}] datanode.DataXceiver(331): 127.0.0.1:35007:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:47296 dst: /127.0.0.1:35007
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:35:28,283 WARN  [Thread-932 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35683,DS-71cd9595-34b3-4e78-8dba-8b432c0955e2,DISK]
2024-11-17T20:35:28,284 WARN  [Thread-932 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741850_1033
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:28,285 WARN  [Thread-932 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1477976244-172.17.0.2-1731875704920:blk_1073741850_1033 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42609,DS-59ce2296-6caf-48fd-a174-2d36e11610d2,DISK], DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42609,DS-59ce2296-6caf-48fd-a174-2d36e11610d2,DISK]) is bad.
2024-11-17T20:35:28,285 WARN  [Thread-932 {}] hdfs.DataStreamer(1850): Abandoning BP-1477976244-172.17.0.2-1731875704920:blk_1073741850_1033
2024-11-17T20:35:28,285 WARN  [Thread-932 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42609,DS-59ce2296-6caf-48fd-a174-2d36e11610d2,DISK]
2024-11-17T20:35:28,286 WARN  [Thread-932 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741851_1034
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:28,286 WARN  [Thread-932 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1477976244-172.17.0.2-1731875704920:blk_1073741851_1034 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK], DatanodeInfoWithStorage[127.0.0.1:35007,DS-e3ca24df-f174-47d5-ac27-0b38fd48e050,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]) is bad.
2024-11-17T20:35:28,287 WARN  [Thread-932 {}] hdfs.DataStreamer(1850): Abandoning BP-1477976244-172.17.0.2-1731875704920:blk_1073741851_1034
2024-11-17T20:35:28,287 WARN  [Thread-932 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]
2024-11-17T20:35:28,289 WARN  [Thread-932 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741852_1035
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:28,289 WARN  [Thread-932 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1477976244-172.17.0.2-1731875704920:blk_1073741852_1035 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38211,DS-c88e28b9-4de5-4077-a5d2-a5f42ad2b57f,DISK], DatanodeInfoWithStorage[127.0.0.1:35007,DS-e3ca24df-f174-47d5-ac27-0b38fd48e050,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38211,DS-c88e28b9-4de5-4077-a5d2-a5f42ad2b57f,DISK]) is bad.
2024-11-17T20:35:28,289 WARN  [Thread-932 {}] hdfs.DataStreamer(1850): Abandoning BP-1477976244-172.17.0.2-1731875704920:blk_1073741852_1035
2024-11-17T20:35:28,290 WARN  [Thread-932 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38211,DS-c88e28b9-4de5-4077-a5d2-a5f42ad2b57f,DISK]
2024-11-17T20:35:28,291 WARN  [IPC Server handler 0 on default port 33551 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-17T20:35:28,291 WARN  [IPC Server handler 0 on default port 33551 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-17T20:35:28,291 WARN  [IPC Server handler 0 on default port 33551 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-17T20:35:28,294 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35007 is added to blk_1073741853_1036 (size=12506)
2024-11-17T20:35:28,695 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.50 KB at sequenceid=24 (bloomFilter=true), to=hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/.tmp/info/a633fbb8c8994446b4e51b16e3c0a02d
2024-11-17T20:35:28,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/.tmp/info/a633fbb8c8994446b4e51b16e3c0a02d as hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/info/a633fbb8c8994446b4e51b16e3c0a02d
2024-11-17T20:35:28,712 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/info/a633fbb8c8994446b4e51b16e3c0a02d, entries=7, sequenceid=24, filesize=12.2 K
2024-11-17T20:35:28,714 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~10.50 KB/10756, heapSize ~11.48 KB/11760, currentSize=2.10 KB/2150 for 6ea106502026e091c4a7f9fa49641279 in 440ms, sequenceid=24, compaction requested=false
2024-11-17T20:35:28,714 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ea106502026e091c4a7f9fa49641279:

2024-11-17T20:35:28,714 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=22.3 K, sizeToCheck=16.0 K
2024-11-17T20:35:28,714 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-17T20:35:28,714 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/info/a633fbb8c8994446b4e51b16e3c0a02d because midkey is the same as first or last row
2024-11-17T20:35:29,153 INFO  [regionserver/0b0b0ae8de03:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:29,564 WARN  [regionserver/0b0b0ae8de03:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:35007,DS-e3ca24df-f174-47d5-ac27-0b38fd48e050,DISK]]
2024-11-17T20:35:29,564 INFO  [regionserver/0b0b0ae8de03:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:29,564 DEBUG [regionserver/0b0b0ae8de03:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 0b0b0ae8de03%2C33133%2C1731875707434:(num 1731875725542) roll requested
2024-11-17T20:35:29,565 INFO  [regionserver/0b0b0ae8de03:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0b0b0ae8de03%2C33133%2C1731875707434.1731875729565
2024-11-17T20:35:29,571 WARN  [Thread-937 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741854_1037
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:29,571 WARN  [Thread-937 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1477976244-172.17.0.2-1731875704920:blk_1073741854_1037 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK], DatanodeInfoWithStorage[127.0.0.1:35683,DS-71cd9595-34b3-4e78-8dba-8b432c0955e2,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]) is bad.
2024-11-17T20:35:29,571 WARN  [Thread-937 {}] hdfs.DataStreamer(1850): Abandoning BP-1477976244-172.17.0.2-1731875704920:blk_1073741854_1037
2024-11-17T20:35:29,572 WARN  [Thread-937 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]
2024-11-17T20:35:29,573 WARN  [Thread-937 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741855_1038
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:29,574 WARN  [Thread-937 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1477976244-172.17.0.2-1731875704920:blk_1073741855_1038 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35683,DS-71cd9595-34b3-4e78-8dba-8b432c0955e2,DISK], DatanodeInfoWithStorage[127.0.0.1:38211,DS-c88e28b9-4de5-4077-a5d2-a5f42ad2b57f,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35683,DS-71cd9595-34b3-4e78-8dba-8b432c0955e2,DISK]) is bad.
2024-11-17T20:35:29,574 WARN  [Thread-937 {}] hdfs.DataStreamer(1850): Abandoning BP-1477976244-172.17.0.2-1731875704920:blk_1073741855_1038
2024-11-17T20:35:29,574 WARN  [Thread-937 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35683,DS-71cd9595-34b3-4e78-8dba-8b432c0955e2,DISK]
2024-11-17T20:35:29,577 WARN  [Thread-937 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741856_1039
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:42609
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:29,577 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-723733033_22 at /127.0.0.1:47310 [Receiving block BP-1477976244-172.17.0.2-1731875704920:blk_1073741856_1039] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/cluster_b9ca40e8-4224-d8ab-7466-d1512b745e25/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/cluster_b9ca40e8-4224-d8ab-7466-d1512b745e25/data/data10]'}, localName='127.0.0.1:35007', datanodeUuid='96f27179-3395-467d-8cec-ee4e6e5e306e', xmitsInProgress=0}:Exception transferring block BP-1477976244-172.17.0.2-1731875704920:blk_1073741856_1039 to mirror 127.0.0.1:42609
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:35:29,577 WARN  [Thread-937 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1477976244-172.17.0.2-1731875704920:blk_1073741856_1039 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35007,DS-e3ca24df-f174-47d5-ac27-0b38fd48e050,DISK], DatanodeInfoWithStorage[127.0.0.1:42609,DS-59ce2296-6caf-48fd-a174-2d36e11610d2,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:42609,DS-59ce2296-6caf-48fd-a174-2d36e11610d2,DISK]) is bad.
2024-11-17T20:35:29,577 WARN  [Thread-937 {}] hdfs.DataStreamer(1850): Abandoning BP-1477976244-172.17.0.2-1731875704920:blk_1073741856_1039
2024-11-17T20:35:29,577 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-723733033_22 at /127.0.0.1:47310 [Receiving block BP-1477976244-172.17.0.2-1731875704920:blk_1073741856_1039] {}] datanode.BlockReceiver(316): Block 1073741856 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-17T20:35:29,577 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-723733033_22 at /127.0.0.1:47310 [Receiving block BP-1477976244-172.17.0.2-1731875704920:blk_1073741856_1039] {}] datanode.DataXceiver(331): 127.0.0.1:35007:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:47310 dst: /127.0.0.1:35007
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:35:29,578 WARN  [Thread-937 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42609,DS-59ce2296-6caf-48fd-a174-2d36e11610d2,DISK]
2024-11-17T20:35:29,580 WARN  [Thread-937 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741857_1040
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:38211
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:29,581 WARN  [Thread-937 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1477976244-172.17.0.2-1731875704920:blk_1073741857_1040 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35007,DS-e3ca24df-f174-47d5-ac27-0b38fd48e050,DISK], DatanodeInfoWithStorage[127.0.0.1:38211,DS-c88e28b9-4de5-4077-a5d2-a5f42ad2b57f,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:38211,DS-c88e28b9-4de5-4077-a5d2-a5f42ad2b57f,DISK]) is bad.
2024-11-17T20:35:29,580 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-723733033_22 at /127.0.0.1:47316 [Receiving block BP-1477976244-172.17.0.2-1731875704920:blk_1073741857_1040] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/cluster_b9ca40e8-4224-d8ab-7466-d1512b745e25/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/cluster_b9ca40e8-4224-d8ab-7466-d1512b745e25/data/data10]'}, localName='127.0.0.1:35007', datanodeUuid='96f27179-3395-467d-8cec-ee4e6e5e306e', xmitsInProgress=0}:Exception transferring block BP-1477976244-172.17.0.2-1731875704920:blk_1073741857_1040 to mirror 127.0.0.1:38211
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:35:29,581 WARN  [Thread-937 {}] hdfs.DataStreamer(1850): Abandoning BP-1477976244-172.17.0.2-1731875704920:blk_1073741857_1040
2024-11-17T20:35:29,581 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-723733033_22 at /127.0.0.1:47316 [Receiving block BP-1477976244-172.17.0.2-1731875704920:blk_1073741857_1040] {}] datanode.BlockReceiver(316): Block 1073741857 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-17T20:35:29,581 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-723733033_22 at /127.0.0.1:47316 [Receiving block BP-1477976244-172.17.0.2-1731875704920:blk_1073741857_1040] {}] datanode.DataXceiver(331): 127.0.0.1:35007:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:47316 dst: /127.0.0.1:35007
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:35:29,581 WARN  [Thread-937 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38211,DS-c88e28b9-4de5-4077-a5d2-a5f42ad2b57f,DISK]
2024-11-17T20:35:29,582 WARN  [IPC Server handler 1 on default port 33551 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-17T20:35:29,582 WARN  [IPC Server handler 1 on default port 33551 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-17T20:35:29,582 WARN  [IPC Server handler 1 on default port 33551 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-17T20:35:29,585 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:35:29,585 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:35:29,585 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:35:29,585 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:35:29,585 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:35:29,586 INFO  [regionserver/0b0b0ae8de03:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.1731875725542 with entries=25, filesize=25.38 KB; new WAL /user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.1731875729565
2024-11-17T20:35:29,588 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35007 is added to blk_1073741843_1026 (size=25992)
2024-11-17T20:35:29,593 DEBUG [regionserver/0b0b0ae8de03:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34837:34837)]
2024-11-17T20:35:29,593 DEBUG [regionserver/0b0b0ae8de03:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.1731875708280 is not closed yet, will try archiving it next time
2024-11-17T20:35:29,593 DEBUG [regionserver/0b0b0ae8de03:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.1731875725542 is not closed yet, will try archiving it next time
2024-11-17T20:35:29,595 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.1731875721525 to hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/oldWALs/0b0b0ae8de03%2C33133%2C1731875707434.1731875721525
2024-11-17T20:35:29,619 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:29,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33133 {}] regionserver.HRegion(8855): Flush requested on 6ea106502026e091c4a7f9fa49641279
2024-11-17T20:35:29,693 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ea106502026e091c4a7f9fa49641279 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-11-17T20:35:29,699 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/.tmp/info/3071f7c88b2e44749b1d0c1483a5c9f2 is 1079, key is tmprow/info:/1731875729691/Put/seqid=0
2024-11-17T20:35:29,701 WARN  [Thread-943 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741859_1042
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:29,701 WARN  [Thread-943 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1477976244-172.17.0.2-1731875704920:blk_1073741859_1042 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42609,DS-59ce2296-6caf-48fd-a174-2d36e11610d2,DISK], DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42609,DS-59ce2296-6caf-48fd-a174-2d36e11610d2,DISK]) is bad.
2024-11-17T20:35:29,701 WARN  [Thread-943 {}] hdfs.DataStreamer(1850): Abandoning BP-1477976244-172.17.0.2-1731875704920:blk_1073741859_1042
2024-11-17T20:35:29,702 WARN  [Thread-943 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42609,DS-59ce2296-6caf-48fd-a174-2d36e11610d2,DISK]
2024-11-17T20:35:29,704 WARN  [Thread-943 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741860_1043
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:29,704 WARN  [Thread-943 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1477976244-172.17.0.2-1731875704920:blk_1073741860_1043 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35683,DS-71cd9595-34b3-4e78-8dba-8b432c0955e2,DISK], DatanodeInfoWithStorage[127.0.0.1:38211,DS-c88e28b9-4de5-4077-a5d2-a5f42ad2b57f,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35683,DS-71cd9595-34b3-4e78-8dba-8b432c0955e2,DISK]) is bad.
2024-11-17T20:35:29,704 WARN  [Thread-943 {}] hdfs.DataStreamer(1850): Abandoning BP-1477976244-172.17.0.2-1731875704920:blk_1073741860_1043
2024-11-17T20:35:29,705 WARN  [Thread-943 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35683,DS-71cd9595-34b3-4e78-8dba-8b432c0955e2,DISK]
2024-11-17T20:35:29,709 WARN  [Thread-943 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741861_1044
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:41145
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:29,709 WARN  [Thread-943 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1477976244-172.17.0.2-1731875704920:blk_1073741861_1044 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35007,DS-e3ca24df-f174-47d5-ac27-0b38fd48e050,DISK], DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]) is bad.
2024-11-17T20:35:29,709 WARN  [Thread-943 {}] hdfs.DataStreamer(1850): Abandoning BP-1477976244-172.17.0.2-1731875704920:blk_1073741861_1044
2024-11-17T20:35:29,710 WARN  [Thread-943 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]
2024-11-17T20:35:29,709 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-723733033_22 at /127.0.0.1:47324 [Receiving block BP-1477976244-172.17.0.2-1731875704920:blk_1073741861_1044] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/cluster_b9ca40e8-4224-d8ab-7466-d1512b745e25/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/cluster_b9ca40e8-4224-d8ab-7466-d1512b745e25/data/data10]'}, localName='127.0.0.1:35007', datanodeUuid='96f27179-3395-467d-8cec-ee4e6e5e306e', xmitsInProgress=0}:Exception transferring block BP-1477976244-172.17.0.2-1731875704920:blk_1073741861_1044 to mirror 127.0.0.1:41145
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:35:29,714 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-723733033_22 at /127.0.0.1:47324 [Receiving block BP-1477976244-172.17.0.2-1731875704920:blk_1073741861_1044] {}] datanode.BlockReceiver(316): Block 1073741861 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-17T20:35:29,714 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-723733033_22 at /127.0.0.1:47324 [Receiving block BP-1477976244-172.17.0.2-1731875704920:blk_1073741861_1044] {}] datanode.DataXceiver(331): 127.0.0.1:35007:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:47324 dst: /127.0.0.1:35007
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:35:29,717 WARN  [Thread-943 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741862_1045
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:38211
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:29,717 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-723733033_22 at /127.0.0.1:47332 [Receiving block BP-1477976244-172.17.0.2-1731875704920:blk_1073741862_1045] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/cluster_b9ca40e8-4224-d8ab-7466-d1512b745e25/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/cluster_b9ca40e8-4224-d8ab-7466-d1512b745e25/data/data10]'}, localName='127.0.0.1:35007', datanodeUuid='96f27179-3395-467d-8cec-ee4e6e5e306e', xmitsInProgress=0}:Exception transferring block BP-1477976244-172.17.0.2-1731875704920:blk_1073741862_1045 to mirror 127.0.0.1:38211
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:35:29,718 WARN  [Thread-943 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1477976244-172.17.0.2-1731875704920:blk_1073741862_1045 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35007,DS-e3ca24df-f174-47d5-ac27-0b38fd48e050,DISK], DatanodeInfoWithStorage[127.0.0.1:38211,DS-c88e28b9-4de5-4077-a5d2-a5f42ad2b57f,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:38211,DS-c88e28b9-4de5-4077-a5d2-a5f42ad2b57f,DISK]) is bad.
2024-11-17T20:35:29,718 WARN  [Thread-943 {}] hdfs.DataStreamer(1850): Abandoning BP-1477976244-172.17.0.2-1731875704920:blk_1073741862_1045
2024-11-17T20:35:29,718 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-723733033_22 at /127.0.0.1:47332 [Receiving block BP-1477976244-172.17.0.2-1731875704920:blk_1073741862_1045] {}] datanode.BlockReceiver(316): Block 1073741862 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-17T20:35:29,718 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-723733033_22 at /127.0.0.1:47332 [Receiving block BP-1477976244-172.17.0.2-1731875704920:blk_1073741862_1045] {}] datanode.DataXceiver(331): 127.0.0.1:35007:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:47332 dst: /127.0.0.1:35007
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:35:29,718 WARN  [Thread-943 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38211,DS-c88e28b9-4de5-4077-a5d2-a5f42ad2b57f,DISK]
2024-11-17T20:35:29,719 WARN  [IPC Server handler 4 on default port 33551 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-17T20:35:29,719 WARN  [IPC Server handler 4 on default port 33551 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-17T20:35:29,720 WARN  [IPC Server handler 4 on default port 33551 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-17T20:35:29,724 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35007 is added to blk_1073741863_1046 (size=6027)
2024-11-17T20:35:29,989 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.1731875708280 is not closed yet, will try archiving it next time
2024-11-17T20:35:30,125 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/.tmp/info/3071f7c88b2e44749b1d0c1483a5c9f2
2024-11-17T20:35:30,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/.tmp/info/3071f7c88b2e44749b1d0c1483a5c9f2 as hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/info/3071f7c88b2e44749b1d0c1483a5c9f2
2024-11-17T20:35:30,145 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/info/3071f7c88b2e44749b1d0c1483a5c9f2, entries=1, sequenceid=34, filesize=5.9 K
2024-11-17T20:35:30,146 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for 6ea106502026e091c4a7f9fa49641279 in 453ms, sequenceid=34, compaction requested=true
2024-11-17T20:35:30,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ea106502026e091c4a7f9fa49641279:

2024-11-17T20:35:30,147 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=28.2 K, sizeToCheck=16.0 K
2024-11-17T20:35:30,147 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-17T20:35:30,147 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/info/a633fbb8c8994446b4e51b16e3c0a02d because midkey is the same as first or last row
2024-11-17T20:35:30,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ea106502026e091c4a7f9fa49641279:info, priority=-2147483648, current under compaction store size is 1
2024-11-17T20:35:30,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-17T20:35:30,147 DEBUG [RS:0;0b0b0ae8de03:33133-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-17T20:35:30,149 DEBUG [RS:0;0b0b0ae8de03:33133-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 28880 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-17T20:35:30,149 DEBUG [RS:0;0b0b0ae8de03:33133-shortCompactions-0 {}] regionserver.HStore(1541): 6ea106502026e091c4a7f9fa49641279/info is initiating minor compaction (all files)
2024-11-17T20:35:30,149 INFO  [RS:0;0b0b0ae8de03:33133-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ea106502026e091c4a7f9fa49641279/info in TestLogRolling-testLogRollOnDatanodeDeath,,1731875709192.6ea106502026e091c4a7f9fa49641279.
2024-11-17T20:35:30,149 INFO  [RS:0;0b0b0ae8de03:33133-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/info/3f1fe0c9aab24b899b5dfe69f3f5857c, hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/info/a633fbb8c8994446b4e51b16e3c0a02d, hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/info/3071f7c88b2e44749b1d0c1483a5c9f2] into tmpdir=hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/.tmp, totalSize=28.2 K
2024-11-17T20:35:30,150 DEBUG [RS:0;0b0b0ae8de03:33133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f1fe0c9aab24b899b5dfe69f3f5857c, keycount=5, bloomtype=ROW, size=10.1 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731875723584
2024-11-17T20:35:30,151 DEBUG [RS:0;0b0b0ae8de03:33133-shortCompactions-0 {}] compactions.Compactor(225): Compacting a633fbb8c8994446b4e51b16e3c0a02d, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=24, earliestPutTs=1731875727644
2024-11-17T20:35:30,151 DEBUG [RS:0;0b0b0ae8de03:33133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3071f7c88b2e44749b1d0c1483a5c9f2, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1731875729691
2024-11-17T20:35:30,180 INFO  [RS:0;0b0b0ae8de03:33133-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ea106502026e091c4a7f9fa49641279#info#compaction#21 average throughput is 4.10 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-17T20:35:30,181 DEBUG [RS:0;0b0b0ae8de03:33133-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/.tmp/info/234c08f58e3642d895af9af2ea82de0a is 1080, key is row0002/info:/1731875723584/Put/seqid=0
2024-11-17T20:35:30,186 WARN  [Thread-951 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741864_1047
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:30,187 WARN  [Thread-951 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1477976244-172.17.0.2-1731875704920:blk_1073741864_1047 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35683,DS-71cd9595-34b3-4e78-8dba-8b432c0955e2,DISK], DatanodeInfoWithStorage[127.0.0.1:42609,DS-59ce2296-6caf-48fd-a174-2d36e11610d2,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35683,DS-71cd9595-34b3-4e78-8dba-8b432c0955e2,DISK]) is bad.
2024-11-17T20:35:30,187 WARN  [Thread-951 {}] hdfs.DataStreamer(1850): Abandoning BP-1477976244-172.17.0.2-1731875704920:blk_1073741864_1047
2024-11-17T20:35:30,188 WARN  [Thread-951 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35683,DS-71cd9595-34b3-4e78-8dba-8b432c0955e2,DISK]
2024-11-17T20:35:30,190 WARN  [Thread-951 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741865_1048
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:30,190 WARN  [Thread-951 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1477976244-172.17.0.2-1731875704920:blk_1073741865_1048 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK], DatanodeInfoWithStorage[127.0.0.1:42609,DS-59ce2296-6caf-48fd-a174-2d36e11610d2,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]) is bad.
2024-11-17T20:35:30,190 WARN  [Thread-951 {}] hdfs.DataStreamer(1850): Abandoning BP-1477976244-172.17.0.2-1731875704920:blk_1073741865_1048
2024-11-17T20:35:30,191 WARN  [Thread-951 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]
2024-11-17T20:35:30,193 WARN  [Thread-951 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741866_1049
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:30,194 WARN  [Thread-951 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1477976244-172.17.0.2-1731875704920:blk_1073741866_1049 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42609,DS-59ce2296-6caf-48fd-a174-2d36e11610d2,DISK], DatanodeInfoWithStorage[127.0.0.1:35007,DS-e3ca24df-f174-47d5-ac27-0b38fd48e050,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42609,DS-59ce2296-6caf-48fd-a174-2d36e11610d2,DISK]) is bad.
2024-11-17T20:35:30,194 WARN  [Thread-951 {}] hdfs.DataStreamer(1850): Abandoning BP-1477976244-172.17.0.2-1731875704920:blk_1073741866_1049
2024-11-17T20:35:30,195 WARN  [Thread-951 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42609,DS-59ce2296-6caf-48fd-a174-2d36e11610d2,DISK]
2024-11-17T20:35:30,196 WARN  [Thread-951 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741867_1050
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:30,196 WARN  [Thread-951 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1477976244-172.17.0.2-1731875704920:blk_1073741867_1050 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38211,DS-c88e28b9-4de5-4077-a5d2-a5f42ad2b57f,DISK], DatanodeInfoWithStorage[127.0.0.1:35007,DS-e3ca24df-f174-47d5-ac27-0b38fd48e050,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38211,DS-c88e28b9-4de5-4077-a5d2-a5f42ad2b57f,DISK]) is bad.
2024-11-17T20:35:30,196 WARN  [Thread-951 {}] hdfs.DataStreamer(1850): Abandoning BP-1477976244-172.17.0.2-1731875704920:blk_1073741867_1050
2024-11-17T20:35:30,197 WARN  [Thread-951 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38211,DS-c88e28b9-4de5-4077-a5d2-a5f42ad2b57f,DISK]
2024-11-17T20:35:30,198 WARN  [IPC Server handler 2 on default port 33551 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-17T20:35:30,198 WARN  [IPC Server handler 2 on default port 33551 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-17T20:35:30,198 WARN  [IPC Server handler 2 on default port 33551 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-17T20:35:30,210 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35007 is added to blk_1073741868_1051 (size=17994)
2024-11-17T20:35:30,425 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@45975e5[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:35007, datanodeUuid=96f27179-3395-467d-8cec-ee4e6e5e306e, infoPort=34837, infoSecurePort=0, ipcPort=46245, storageInfo=lv=-57;cid=testClusterID;nsid=991965062;c=1731875704920):Failed to transfer BP-1477976244-172.17.0.2-1731875704920:blk_1073741848_1031 to 127.0.0.1:42609 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:35:30,425 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@66f1b6[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:35007, datanodeUuid=96f27179-3395-467d-8cec-ee4e6e5e306e, infoPort=34837, infoSecurePort=0, ipcPort=46245, storageInfo=lv=-57;cid=testClusterID;nsid=991965062;c=1731875704920):Failed to transfer BP-1477976244-172.17.0.2-1731875704920:blk_1073741853_1036 to 127.0.0.1:41145 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:35:30,619 DEBUG [RS:0;0b0b0ae8de03:33133-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/.tmp/info/234c08f58e3642d895af9af2ea82de0a as hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/info/234c08f58e3642d895af9af2ea82de0a
2024-11-17T20:35:30,628 INFO  [RS:0;0b0b0ae8de03:33133-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ea106502026e091c4a7f9fa49641279/info of 6ea106502026e091c4a7f9fa49641279 into 234c08f58e3642d895af9af2ea82de0a(size=17.6 K), total size for store is 17.6 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-17T20:35:30,628 DEBUG [RS:0;0b0b0ae8de03:33133-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ea106502026e091c4a7f9fa49641279:

2024-11-17T20:35:30,628 INFO  [RS:0;0b0b0ae8de03:33133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1731875709192.6ea106502026e091c4a7f9fa49641279., storeName=6ea106502026e091c4a7f9fa49641279/info, priority=13, startTime=1731875730147; duration=0sec
2024-11-17T20:35:30,628 DEBUG [RS:0;0b0b0ae8de03:33133-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K
2024-11-17T20:35:30,628 DEBUG [RS:0;0b0b0ae8de03:33133-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-17T20:35:30,628 DEBUG [RS:0;0b0b0ae8de03:33133-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/info/234c08f58e3642d895af9af2ea82de0a because midkey is the same as first or last row
2024-11-17T20:35:30,629 DEBUG [RS:0;0b0b0ae8de03:33133-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K
2024-11-17T20:35:30,629 DEBUG [RS:0;0b0b0ae8de03:33133-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-17T20:35:30,629 DEBUG [RS:0;0b0b0ae8de03:33133-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/info/234c08f58e3642d895af9af2ea82de0a because midkey is the same as first or last row
2024-11-17T20:35:30,629 DEBUG [RS:0;0b0b0ae8de03:33133-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K
2024-11-17T20:35:30,629 DEBUG [RS:0;0b0b0ae8de03:33133-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-17T20:35:30,629 DEBUG [RS:0;0b0b0ae8de03:33133-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/info/234c08f58e3642d895af9af2ea82de0a because midkey is the same as first or last row
2024-11-17T20:35:30,629 DEBUG [RS:0;0b0b0ae8de03:33133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-17T20:35:30,629 DEBUG [RS:0;0b0b0ae8de03:33133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ea106502026e091c4a7f9fa49641279:info
2024-11-17T20:35:31,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33133 {}] regionserver.HRegion(8855): Flush requested on 6ea106502026e091c4a7f9fa49641279
2024-11-17T20:35:31,116 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ea106502026e091c4a7f9fa49641279 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-11-17T20:35:31,121 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/.tmp/info/1f46bd98a1db4ac1aa38719addc502bf is 1079, key is tmprow/info:/1731875731114/Put/seqid=0
2024-11-17T20:35:31,124 WARN  [Thread-958 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741869_1052
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:35683
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:31,124 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-723733033_22 at /127.0.0.1:47384 [Receiving block BP-1477976244-172.17.0.2-1731875704920:blk_1073741869_1052] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/cluster_b9ca40e8-4224-d8ab-7466-d1512b745e25/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/cluster_b9ca40e8-4224-d8ab-7466-d1512b745e25/data/data10]'}, localName='127.0.0.1:35007', datanodeUuid='96f27179-3395-467d-8cec-ee4e6e5e306e', xmitsInProgress=0}:Exception transferring block BP-1477976244-172.17.0.2-1731875704920:blk_1073741869_1052 to mirror 127.0.0.1:35683
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:35:31,124 WARN  [Thread-958 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1477976244-172.17.0.2-1731875704920:blk_1073741869_1052 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35007,DS-e3ca24df-f174-47d5-ac27-0b38fd48e050,DISK], DatanodeInfoWithStorage[127.0.0.1:35683,DS-71cd9595-34b3-4e78-8dba-8b432c0955e2,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:35683,DS-71cd9595-34b3-4e78-8dba-8b432c0955e2,DISK]) is bad.
2024-11-17T20:35:31,125 WARN  [Thread-958 {}] hdfs.DataStreamer(1850): Abandoning BP-1477976244-172.17.0.2-1731875704920:blk_1073741869_1052
2024-11-17T20:35:31,125 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-723733033_22 at /127.0.0.1:47384 [Receiving block BP-1477976244-172.17.0.2-1731875704920:blk_1073741869_1052] {}] datanode.BlockReceiver(316): Block 1073741869 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-17T20:35:31,125 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-723733033_22 at /127.0.0.1:47384 [Receiving block BP-1477976244-172.17.0.2-1731875704920:blk_1073741869_1052] {}] datanode.DataXceiver(331): 127.0.0.1:35007:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:47384 dst: /127.0.0.1:35007
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:35:31,125 WARN  [Thread-958 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35683,DS-71cd9595-34b3-4e78-8dba-8b432c0955e2,DISK]
2024-11-17T20:35:31,126 WARN  [Thread-958 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741870_1053
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:31,127 WARN  [Thread-958 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1477976244-172.17.0.2-1731875704920:blk_1073741870_1053 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42609,DS-59ce2296-6caf-48fd-a174-2d36e11610d2,DISK], DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42609,DS-59ce2296-6caf-48fd-a174-2d36e11610d2,DISK]) is bad.
2024-11-17T20:35:31,127 WARN  [Thread-958 {}] hdfs.DataStreamer(1850): Abandoning BP-1477976244-172.17.0.2-1731875704920:blk_1073741870_1053
2024-11-17T20:35:31,127 WARN  [Thread-958 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42609,DS-59ce2296-6caf-48fd-a174-2d36e11610d2,DISK]
2024-11-17T20:35:31,129 WARN  [Thread-958 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741871_1054
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:31,129 WARN  [Thread-958 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1477976244-172.17.0.2-1731875704920:blk_1073741871_1054 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38211,DS-c88e28b9-4de5-4077-a5d2-a5f42ad2b57f,DISK], DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38211,DS-c88e28b9-4de5-4077-a5d2-a5f42ad2b57f,DISK]) is bad.
2024-11-17T20:35:31,129 WARN  [Thread-958 {}] hdfs.DataStreamer(1850): Abandoning BP-1477976244-172.17.0.2-1731875704920:blk_1073741871_1054
2024-11-17T20:35:31,130 WARN  [Thread-958 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38211,DS-c88e28b9-4de5-4077-a5d2-a5f42ad2b57f,DISK]
2024-11-17T20:35:31,132 WARN  [Thread-958 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741872_1055
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:31,132 WARN  [Thread-958 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1477976244-172.17.0.2-1731875704920:blk_1073741872_1055 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK], DatanodeInfoWithStorage[127.0.0.1:35007,DS-e3ca24df-f174-47d5-ac27-0b38fd48e050,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]) is bad.
2024-11-17T20:35:31,132 WARN  [Thread-958 {}] hdfs.DataStreamer(1850): Abandoning BP-1477976244-172.17.0.2-1731875704920:blk_1073741872_1055
2024-11-17T20:35:31,133 WARN  [Thread-958 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]
2024-11-17T20:35:31,134 WARN  [IPC Server handler 1 on default port 33551 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-17T20:35:31,134 WARN  [IPC Server handler 1 on default port 33551 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-17T20:35:31,135 WARN  [IPC Server handler 1 on default port 33551 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-17T20:35:31,151 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35007 is added to blk_1073741873_1056 (size=6027)
2024-11-17T20:35:31,154 INFO  [regionserver/0b0b0ae8de03:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:31,422 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@66f1b6[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:35007, datanodeUuid=96f27179-3395-467d-8cec-ee4e6e5e306e, infoPort=34837, infoSecurePort=0, ipcPort=46245, storageInfo=lv=-57;cid=testClusterID;nsid=991965062;c=1731875704920):Failed to transfer BP-1477976244-172.17.0.2-1731875704920:blk_1073741843_1026 to 127.0.0.1:42609 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:35:31,552 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/.tmp/info/1f46bd98a1db4ac1aa38719addc502bf
2024-11-17T20:35:31,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/.tmp/info/1f46bd98a1db4ac1aa38719addc502bf as hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/info/1f46bd98a1db4ac1aa38719addc502bf
2024-11-17T20:35:31,568 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/info/1f46bd98a1db4ac1aa38719addc502bf, entries=1, sequenceid=45, filesize=5.9 K
2024-11-17T20:35:31,569 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for 6ea106502026e091c4a7f9fa49641279 in 453ms, sequenceid=45, compaction requested=false
2024-11-17T20:35:31,569 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ea106502026e091c4a7f9fa49641279:

2024-11-17T20:35:31,569 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=23.5 K, sizeToCheck=16.0 K
2024-11-17T20:35:31,569 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-17T20:35:31,570 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/info/234c08f58e3642d895af9af2ea82de0a because midkey is the same as first or last row
2024-11-17T20:35:31,595 WARN  [regionserver/0b0b0ae8de03:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:35007,DS-e3ca24df-f174-47d5-ac27-0b38fd48e050,DISK]]
2024-11-17T20:35:31,595 INFO  [regionserver/0b0b0ae8de03:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:31,595 DEBUG [regionserver/0b0b0ae8de03:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 0b0b0ae8de03%2C33133%2C1731875707434:(num 1731875729565) roll requested
2024-11-17T20:35:31,596 INFO  [regionserver/0b0b0ae8de03:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0b0b0ae8de03%2C33133%2C1731875707434.1731875731596
2024-11-17T20:35:31,598 WARN  [Thread-963 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741874_1057
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:31,599 WARN  [Thread-963 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1477976244-172.17.0.2-1731875704920:blk_1073741874_1057 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK], DatanodeInfoWithStorage[127.0.0.1:35683,DS-71cd9595-34b3-4e78-8dba-8b432c0955e2,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]) is bad.
2024-11-17T20:35:31,599 WARN  [Thread-963 {}] hdfs.DataStreamer(1850): Abandoning BP-1477976244-172.17.0.2-1731875704920:blk_1073741874_1057
2024-11-17T20:35:31,599 WARN  [Thread-963 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]
2024-11-17T20:35:31,601 WARN  [Thread-963 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741875_1058
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:31,601 WARN  [Thread-963 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1477976244-172.17.0.2-1731875704920:blk_1073741875_1058 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35683,DS-71cd9595-34b3-4e78-8dba-8b432c0955e2,DISK], DatanodeInfoWithStorage[127.0.0.1:42609,DS-59ce2296-6caf-48fd-a174-2d36e11610d2,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35683,DS-71cd9595-34b3-4e78-8dba-8b432c0955e2,DISK]) is bad.
2024-11-17T20:35:31,601 WARN  [Thread-963 {}] hdfs.DataStreamer(1850): Abandoning BP-1477976244-172.17.0.2-1731875704920:blk_1073741875_1058
2024-11-17T20:35:31,601 WARN  [Thread-963 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35683,DS-71cd9595-34b3-4e78-8dba-8b432c0955e2,DISK]
2024-11-17T20:35:31,603 WARN  [Thread-963 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741876_1059
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:31,603 WARN  [Thread-963 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1477976244-172.17.0.2-1731875704920:blk_1073741876_1059 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42609,DS-59ce2296-6caf-48fd-a174-2d36e11610d2,DISK], DatanodeInfoWithStorage[127.0.0.1:35007,DS-e3ca24df-f174-47d5-ac27-0b38fd48e050,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42609,DS-59ce2296-6caf-48fd-a174-2d36e11610d2,DISK]) is bad.
2024-11-17T20:35:31,603 WARN  [Thread-963 {}] hdfs.DataStreamer(1850): Abandoning BP-1477976244-172.17.0.2-1731875704920:blk_1073741876_1059
2024-11-17T20:35:31,604 WARN  [Thread-963 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42609,DS-59ce2296-6caf-48fd-a174-2d36e11610d2,DISK]
2024-11-17T20:35:31,606 WARN  [Thread-963 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741877_1060
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:38211
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:31,606 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-723733033_22 at /127.0.0.1:47402 [Receiving block BP-1477976244-172.17.0.2-1731875704920:blk_1073741877_1060] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/cluster_b9ca40e8-4224-d8ab-7466-d1512b745e25/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/cluster_b9ca40e8-4224-d8ab-7466-d1512b745e25/data/data10]'}, localName='127.0.0.1:35007', datanodeUuid='96f27179-3395-467d-8cec-ee4e6e5e306e', xmitsInProgress=0}:Exception transferring block BP-1477976244-172.17.0.2-1731875704920:blk_1073741877_1060 to mirror 127.0.0.1:38211
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:35:31,606 WARN  [Thread-963 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1477976244-172.17.0.2-1731875704920:blk_1073741877_1060 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35007,DS-e3ca24df-f174-47d5-ac27-0b38fd48e050,DISK], DatanodeInfoWithStorage[127.0.0.1:38211,DS-c88e28b9-4de5-4077-a5d2-a5f42ad2b57f,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:38211,DS-c88e28b9-4de5-4077-a5d2-a5f42ad2b57f,DISK]) is bad.
2024-11-17T20:35:31,606 WARN  [Thread-963 {}] hdfs.DataStreamer(1850): Abandoning BP-1477976244-172.17.0.2-1731875704920:blk_1073741877_1060
2024-11-17T20:35:31,606 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-723733033_22 at /127.0.0.1:47402 [Receiving block BP-1477976244-172.17.0.2-1731875704920:blk_1073741877_1060] {}] datanode.BlockReceiver(316): Block 1073741877 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-17T20:35:31,606 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-723733033_22 at /127.0.0.1:47402 [Receiving block BP-1477976244-172.17.0.2-1731875704920:blk_1073741877_1060] {}] datanode.DataXceiver(331): 127.0.0.1:35007:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:47402 dst: /127.0.0.1:35007
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:35:31,607 WARN  [Thread-963 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38211,DS-c88e28b9-4de5-4077-a5d2-a5f42ad2b57f,DISK]
2024-11-17T20:35:31,608 WARN  [IPC Server handler 0 on default port 33551 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-17T20:35:31,608 WARN  [IPC Server handler 0 on default port 33551 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-17T20:35:31,608 WARN  [IPC Server handler 0 on default port 33551 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-17T20:35:31,612 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:35:31,612 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:35:31,612 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:35:31,612 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:35:31,612 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:35:31,612 INFO  [regionserver/0b0b0ae8de03:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.1731875729565 with entries=15, filesize=13.26 KB; new WAL /user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.1731875731596
2024-11-17T20:35:31,613 DEBUG [regionserver/0b0b0ae8de03:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34837:34837)]
2024-11-17T20:35:31,613 DEBUG [regionserver/0b0b0ae8de03:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.1731875708280 is not closed yet, will try archiving it next time
2024-11-17T20:35:31,613 DEBUG [regionserver/0b0b0ae8de03:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.1731875729565 is not closed yet, will try archiving it next time
2024-11-17T20:35:31,613 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.1731875725542 to hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/oldWALs/0b0b0ae8de03%2C33133%2C1731875707434.1731875725542
2024-11-17T20:35:31,614 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35007 is added to blk_1073741858_1041 (size=13591)
2024-11-17T20:35:31,619 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:32,015 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.1731875708280 is not closed yet, will try archiving it next time
2024-11-17T20:35:32,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33133 {}] regionserver.HRegion(8855): Flush requested on 6ea106502026e091c4a7f9fa49641279
2024-11-17T20:35:32,537 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ea106502026e091c4a7f9fa49641279 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-11-17T20:35:32,542 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/.tmp/info/290181b0ca71497bbc71c5f1dc48c46a is 1079, key is tmprow/info:/1731875732535/Put/seqid=0
2024-11-17T20:35:32,545 WARN  [Thread-968 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741879_1062
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:42609
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:32,545 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-723733033_22 at /127.0.0.1:49518 [Receiving block BP-1477976244-172.17.0.2-1731875704920:blk_1073741879_1062] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/cluster_b9ca40e8-4224-d8ab-7466-d1512b745e25/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/cluster_b9ca40e8-4224-d8ab-7466-d1512b745e25/data/data10]'}, localName='127.0.0.1:35007', datanodeUuid='96f27179-3395-467d-8cec-ee4e6e5e306e', xmitsInProgress=0}:Exception transferring block BP-1477976244-172.17.0.2-1731875704920:blk_1073741879_1062 to mirror 127.0.0.1:42609
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:35:32,546 WARN  [Thread-968 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1477976244-172.17.0.2-1731875704920:blk_1073741879_1062 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35007,DS-e3ca24df-f174-47d5-ac27-0b38fd48e050,DISK], DatanodeInfoWithStorage[127.0.0.1:42609,DS-59ce2296-6caf-48fd-a174-2d36e11610d2,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:42609,DS-59ce2296-6caf-48fd-a174-2d36e11610d2,DISK]) is bad.
2024-11-17T20:35:32,546 WARN  [Thread-968 {}] hdfs.DataStreamer(1850): Abandoning BP-1477976244-172.17.0.2-1731875704920:blk_1073741879_1062
2024-11-17T20:35:32,546 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-723733033_22 at /127.0.0.1:49518 [Receiving block BP-1477976244-172.17.0.2-1731875704920:blk_1073741879_1062] {}] datanode.BlockReceiver(316): Block 1073741879 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-17T20:35:32,546 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-723733033_22 at /127.0.0.1:49518 [Receiving block BP-1477976244-172.17.0.2-1731875704920:blk_1073741879_1062] {}] datanode.DataXceiver(331): 127.0.0.1:35007:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:49518 dst: /127.0.0.1:35007
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:35:32,546 WARN  [Thread-968 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42609,DS-59ce2296-6caf-48fd-a174-2d36e11610d2,DISK]
2024-11-17T20:35:32,548 WARN  [Thread-968 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741880_1063
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:32,549 WARN  [Thread-968 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1477976244-172.17.0.2-1731875704920:blk_1073741880_1063 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK], DatanodeInfoWithStorage[127.0.0.1:35007,DS-e3ca24df-f174-47d5-ac27-0b38fd48e050,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]) is bad.
2024-11-17T20:35:32,549 WARN  [Thread-968 {}] hdfs.DataStreamer(1850): Abandoning BP-1477976244-172.17.0.2-1731875704920:blk_1073741880_1063
2024-11-17T20:35:32,549 WARN  [Thread-968 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]
2024-11-17T20:35:32,550 WARN  [Thread-968 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741881_1064
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:32,551 WARN  [Thread-968 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1477976244-172.17.0.2-1731875704920:blk_1073741881_1064 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38211,DS-c88e28b9-4de5-4077-a5d2-a5f42ad2b57f,DISK], DatanodeInfoWithStorage[127.0.0.1:35007,DS-e3ca24df-f174-47d5-ac27-0b38fd48e050,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38211,DS-c88e28b9-4de5-4077-a5d2-a5f42ad2b57f,DISK]) is bad.
2024-11-17T20:35:32,551 WARN  [Thread-968 {}] hdfs.DataStreamer(1850): Abandoning BP-1477976244-172.17.0.2-1731875704920:blk_1073741881_1064
2024-11-17T20:35:32,551 WARN  [Thread-968 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38211,DS-c88e28b9-4de5-4077-a5d2-a5f42ad2b57f,DISK]
2024-11-17T20:35:32,554 WARN  [Thread-968 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741882_1065
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:35683
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:32,554 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-723733033_22 at /127.0.0.1:49522 [Receiving block BP-1477976244-172.17.0.2-1731875704920:blk_1073741882_1065] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/cluster_b9ca40e8-4224-d8ab-7466-d1512b745e25/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/cluster_b9ca40e8-4224-d8ab-7466-d1512b745e25/data/data10]'}, localName='127.0.0.1:35007', datanodeUuid='96f27179-3395-467d-8cec-ee4e6e5e306e', xmitsInProgress=0}:Exception transferring block BP-1477976244-172.17.0.2-1731875704920:blk_1073741882_1065 to mirror 127.0.0.1:35683
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:35:32,554 WARN  [Thread-968 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1477976244-172.17.0.2-1731875704920:blk_1073741882_1065 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35007,DS-e3ca24df-f174-47d5-ac27-0b38fd48e050,DISK], DatanodeInfoWithStorage[127.0.0.1:35683,DS-71cd9595-34b3-4e78-8dba-8b432c0955e2,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:35683,DS-71cd9595-34b3-4e78-8dba-8b432c0955e2,DISK]) is bad.
2024-11-17T20:35:32,554 WARN  [Thread-968 {}] hdfs.DataStreamer(1850): Abandoning BP-1477976244-172.17.0.2-1731875704920:blk_1073741882_1065
2024-11-17T20:35:32,554 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-723733033_22 at /127.0.0.1:49522 [Receiving block BP-1477976244-172.17.0.2-1731875704920:blk_1073741882_1065] {}] datanode.BlockReceiver(316): Block 1073741882 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-17T20:35:32,554 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-723733033_22 at /127.0.0.1:49522 [Receiving block BP-1477976244-172.17.0.2-1731875704920:blk_1073741882_1065] {}] datanode.DataXceiver(331): 127.0.0.1:35007:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:49522 dst: /127.0.0.1:35007
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:35:32,555 WARN  [Thread-968 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35683,DS-71cd9595-34b3-4e78-8dba-8b432c0955e2,DISK]
2024-11-17T20:35:32,555 WARN  [IPC Server handler 3 on default port 33551 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-17T20:35:32,555 WARN  [IPC Server handler 3 on default port 33551 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-17T20:35:32,555 WARN  [IPC Server handler 3 on default port 33551 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-17T20:35:32,558 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35007 is added to blk_1073741883_1066 (size=6027)
2024-11-17T20:35:32,959 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/.tmp/info/290181b0ca71497bbc71c5f1dc48c46a
2024-11-17T20:35:32,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/.tmp/info/290181b0ca71497bbc71c5f1dc48c46a as hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/info/290181b0ca71497bbc71c5f1dc48c46a
2024-11-17T20:35:32,975 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/info/290181b0ca71497bbc71c5f1dc48c46a, entries=1, sequenceid=55, filesize=5.9 K
2024-11-17T20:35:32,976 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for 6ea106502026e091c4a7f9fa49641279 in 440ms, sequenceid=55, compaction requested=true
2024-11-17T20:35:32,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ea106502026e091c4a7f9fa49641279:

2024-11-17T20:35:32,976 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=29.3 K, sizeToCheck=16.0 K
2024-11-17T20:35:32,976 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-17T20:35:32,976 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/info/234c08f58e3642d895af9af2ea82de0a because midkey is the same as first or last row
2024-11-17T20:35:32,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ea106502026e091c4a7f9fa49641279:info, priority=-2147483648, current under compaction store size is 1
2024-11-17T20:35:32,977 DEBUG [RS:0;0b0b0ae8de03:33133-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-17T20:35:32,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-17T20:35:32,978 DEBUG [RS:0;0b0b0ae8de03:33133-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 30048 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-17T20:35:32,978 DEBUG [RS:0;0b0b0ae8de03:33133-shortCompactions-0 {}] regionserver.HStore(1541): 6ea106502026e091c4a7f9fa49641279/info is initiating minor compaction (all files)
2024-11-17T20:35:32,978 INFO  [RS:0;0b0b0ae8de03:33133-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ea106502026e091c4a7f9fa49641279/info in TestLogRolling-testLogRollOnDatanodeDeath,,1731875709192.6ea106502026e091c4a7f9fa49641279.
2024-11-17T20:35:32,978 INFO  [RS:0;0b0b0ae8de03:33133-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/info/234c08f58e3642d895af9af2ea82de0a, hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/info/1f46bd98a1db4ac1aa38719addc502bf, hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/info/290181b0ca71497bbc71c5f1dc48c46a] into tmpdir=hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/.tmp, totalSize=29.3 K
2024-11-17T20:35:32,979 DEBUG [RS:0;0b0b0ae8de03:33133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 234c08f58e3642d895af9af2ea82de0a, keycount=12, bloomtype=ROW, size=17.6 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1731875723584
2024-11-17T20:35:32,979 DEBUG [RS:0;0b0b0ae8de03:33133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f46bd98a1db4ac1aa38719addc502bf, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1731875731114
2024-11-17T20:35:32,980 DEBUG [RS:0;0b0b0ae8de03:33133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 290181b0ca71497bbc71c5f1dc48c46a, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731875732535
2024-11-17T20:35:32,998 INFO  [RS:0;0b0b0ae8de03:33133-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ea106502026e091c4a7f9fa49641279#info#compaction#24 average throughput is 6.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-17T20:35:32,999 DEBUG [RS:0;0b0b0ae8de03:33133-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/.tmp/info/6f1900438fd24cc191be2c8da16b4e9e is 1080, key is row0002/info:/1731875723584/Put/seqid=0
2024-11-17T20:35:33,001 WARN  [Thread-974 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741884_1067
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:33,001 WARN  [Thread-974 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1477976244-172.17.0.2-1731875704920:blk_1073741884_1067 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42609,DS-59ce2296-6caf-48fd-a174-2d36e11610d2,DISK], DatanodeInfoWithStorage[127.0.0.1:35683,DS-71cd9595-34b3-4e78-8dba-8b432c0955e2,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42609,DS-59ce2296-6caf-48fd-a174-2d36e11610d2,DISK]) is bad.
2024-11-17T20:35:33,001 WARN  [Thread-974 {}] hdfs.DataStreamer(1850): Abandoning BP-1477976244-172.17.0.2-1731875704920:blk_1073741884_1067
2024-11-17T20:35:33,002 WARN  [Thread-974 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42609,DS-59ce2296-6caf-48fd-a174-2d36e11610d2,DISK]
2024-11-17T20:35:33,003 WARN  [Thread-974 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741885_1068
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:33,004 WARN  [Thread-974 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1477976244-172.17.0.2-1731875704920:blk_1073741885_1068 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK], DatanodeInfoWithStorage[127.0.0.1:35683,DS-71cd9595-34b3-4e78-8dba-8b432c0955e2,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]) is bad.
2024-11-17T20:35:33,004 WARN  [Thread-974 {}] hdfs.DataStreamer(1850): Abandoning BP-1477976244-172.17.0.2-1731875704920:blk_1073741885_1068
2024-11-17T20:35:33,004 WARN  [Thread-974 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]
2024-11-17T20:35:33,007 WARN  [Thread-974 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741886_1069
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:35683
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:33,007 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-723733033_22 at /127.0.0.1:49550 [Receiving block BP-1477976244-172.17.0.2-1731875704920:blk_1073741886_1069] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/cluster_b9ca40e8-4224-d8ab-7466-d1512b745e25/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/cluster_b9ca40e8-4224-d8ab-7466-d1512b745e25/data/data10]'}, localName='127.0.0.1:35007', datanodeUuid='96f27179-3395-467d-8cec-ee4e6e5e306e', xmitsInProgress=0}:Exception transferring block BP-1477976244-172.17.0.2-1731875704920:blk_1073741886_1069 to mirror 127.0.0.1:35683
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:35:33,007 WARN  [Thread-974 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1477976244-172.17.0.2-1731875704920:blk_1073741886_1069 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35007,DS-e3ca24df-f174-47d5-ac27-0b38fd48e050,DISK], DatanodeInfoWithStorage[127.0.0.1:35683,DS-71cd9595-34b3-4e78-8dba-8b432c0955e2,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:35683,DS-71cd9595-34b3-4e78-8dba-8b432c0955e2,DISK]) is bad.
2024-11-17T20:35:33,007 WARN  [Thread-974 {}] hdfs.DataStreamer(1850): Abandoning BP-1477976244-172.17.0.2-1731875704920:blk_1073741886_1069
2024-11-17T20:35:33,007 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-723733033_22 at /127.0.0.1:49550 [Receiving block BP-1477976244-172.17.0.2-1731875704920:blk_1073741886_1069] {}] datanode.BlockReceiver(316): Block 1073741886 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-17T20:35:33,007 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-723733033_22 at /127.0.0.1:49550 [Receiving block BP-1477976244-172.17.0.2-1731875704920:blk_1073741886_1069] {}] datanode.DataXceiver(331): 127.0.0.1:35007:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:49550 dst: /127.0.0.1:35007
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:35:33,008 WARN  [Thread-974 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35683,DS-71cd9595-34b3-4e78-8dba-8b432c0955e2,DISK]
2024-11-17T20:35:33,009 WARN  [Thread-974 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741887_1070
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:33,009 WARN  [Thread-974 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1477976244-172.17.0.2-1731875704920:blk_1073741887_1070 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38211,DS-c88e28b9-4de5-4077-a5d2-a5f42ad2b57f,DISK], DatanodeInfoWithStorage[127.0.0.1:35007,DS-e3ca24df-f174-47d5-ac27-0b38fd48e050,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38211,DS-c88e28b9-4de5-4077-a5d2-a5f42ad2b57f,DISK]) is bad.
2024-11-17T20:35:33,009 WARN  [Thread-974 {}] hdfs.DataStreamer(1850): Abandoning BP-1477976244-172.17.0.2-1731875704920:blk_1073741887_1070
2024-11-17T20:35:33,010 WARN  [Thread-974 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38211,DS-c88e28b9-4de5-4077-a5d2-a5f42ad2b57f,DISK]
2024-11-17T20:35:33,011 WARN  [IPC Server handler 4 on default port 33551 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-17T20:35:33,011 WARN  [IPC Server handler 4 on default port 33551 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-17T20:35:33,011 WARN  [IPC Server handler 4 on default port 33551 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-17T20:35:33,015 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35007 is added to blk_1073741888_1071 (size=18097)
2024-11-17T20:35:33,154 INFO  [regionserver/0b0b0ae8de03:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:33,422 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@66f1b6[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:35007, datanodeUuid=96f27179-3395-467d-8cec-ee4e6e5e306e, infoPort=34837, infoSecurePort=0, ipcPort=46245, storageInfo=lv=-57;cid=testClusterID;nsid=991965062;c=1731875704920):Failed to transfer BP-1477976244-172.17.0.2-1731875704920:blk_1073741863_1046 to 127.0.0.1:41145 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:35:33,422 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@45975e5[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:35007, datanodeUuid=96f27179-3395-467d-8cec-ee4e6e5e306e, infoPort=34837, infoSecurePort=0, ipcPort=46245, storageInfo=lv=-57;cid=testClusterID;nsid=991965062;c=1731875704920):Failed to transfer BP-1477976244-172.17.0.2-1731875704920:blk_1073741868_1051 to 127.0.0.1:42609 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:35:33,425 DEBUG [RS:0;0b0b0ae8de03:33133-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/.tmp/info/6f1900438fd24cc191be2c8da16b4e9e as hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/info/6f1900438fd24cc191be2c8da16b4e9e
2024-11-17T20:35:33,433 INFO  [RS:0;0b0b0ae8de03:33133-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ea106502026e091c4a7f9fa49641279/info of 6ea106502026e091c4a7f9fa49641279 into 6f1900438fd24cc191be2c8da16b4e9e(size=17.7 K), total size for store is 17.7 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-17T20:35:33,433 DEBUG [RS:0;0b0b0ae8de03:33133-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ea106502026e091c4a7f9fa49641279:

2024-11-17T20:35:33,433 INFO  [RS:0;0b0b0ae8de03:33133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1731875709192.6ea106502026e091c4a7f9fa49641279., storeName=6ea106502026e091c4a7f9fa49641279/info, priority=13, startTime=1731875732977; duration=0sec
2024-11-17T20:35:33,433 DEBUG [RS:0;0b0b0ae8de03:33133-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.7 K, sizeToCheck=16.0 K
2024-11-17T20:35:33,433 DEBUG [RS:0;0b0b0ae8de03:33133-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-17T20:35:33,434 DEBUG [RS:0;0b0b0ae8de03:33133-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/info/6f1900438fd24cc191be2c8da16b4e9e because midkey is the same as first or last row
2024-11-17T20:35:33,434 DEBUG [RS:0;0b0b0ae8de03:33133-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.7 K, sizeToCheck=16.0 K
2024-11-17T20:35:33,434 DEBUG [RS:0;0b0b0ae8de03:33133-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-17T20:35:33,434 DEBUG [RS:0;0b0b0ae8de03:33133-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/info/6f1900438fd24cc191be2c8da16b4e9e because midkey is the same as first or last row
2024-11-17T20:35:33,434 DEBUG [RS:0;0b0b0ae8de03:33133-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.7 K, sizeToCheck=16.0 K
2024-11-17T20:35:33,434 DEBUG [RS:0;0b0b0ae8de03:33133-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-17T20:35:33,434 DEBUG [RS:0;0b0b0ae8de03:33133-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/info/6f1900438fd24cc191be2c8da16b4e9e because midkey is the same as first or last row
2024-11-17T20:35:33,434 DEBUG [RS:0;0b0b0ae8de03:33133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-17T20:35:33,434 DEBUG [RS:0;0b0b0ae8de03:33133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ea106502026e091c4a7f9fa49641279:info
2024-11-17T20:35:33,613 WARN  [regionserver/0b0b0ae8de03:0.logRoller {}] wal.FSHLog(539): Too many consecutive RollWriter requests, it's a sign of the total number of live datanodes is lower than the tolerable replicas.
2024-11-17T20:35:33,614 INFO  [regionserver/0b0b0ae8de03:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:33,620 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:33,770 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-17T20:35:33,774 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-17T20:35:33,775 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-17T20:35:33,775 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-17T20:35:33,775 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-17T20:35:33,775 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1b1c2893{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/hadoop.log.dir/,AVAILABLE}
2024-11-17T20:35:33,775 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@117da513{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-17T20:35:33,877 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3a8cd6ef{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/java.io.tmpdir/jetty-localhost-34629-hadoop-hdfs-3_4_1-tests_jar-_-any-8891432972329335736/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T20:35:33,877 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@25f6ac6c{HTTP/1.1, (http/1.1)}{localhost:34629}
2024-11-17T20:35:33,877 INFO  [Time-limited test {}] server.Server(415): Started @137163ms
2024-11-17T20:35:33,879 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-17T20:35:34,359 WARN  [Thread-993 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-17T20:35:34,370 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd9dd358d3af4b122 with lease ID 0x9923bdc54ecc4197: from storage DS-59ce2296-6caf-48fd-a174-2d36e11610d2 node DatanodeRegistration(127.0.0.1:33949, datanodeUuid=6506fff5-03d5-479e-8b59-d4ac07be411d, infoPort=36805, infoSecurePort=0, ipcPort=42875, storageInfo=lv=-57;cid=testClusterID;nsid=991965062;c=1731875704920), blocks: 7, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-17T20:35:34,371 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd9dd358d3af4b122 with lease ID 0x9923bdc54ecc4197: from storage DS-30f485bb-8766-4446-aa31-1a46b6ed8929 node DatanodeRegistration(127.0.0.1:33949, datanodeUuid=6506fff5-03d5-479e-8b59-d4ac07be411d, infoPort=36805, infoSecurePort=0, ipcPort=42875, storageInfo=lv=-57;cid=testClusterID;nsid=991965062;c=1731875704920), blocks: 6, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-17T20:35:34,423 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@66f1b6[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:35007, datanodeUuid=96f27179-3395-467d-8cec-ee4e6e5e306e, infoPort=34837, infoSecurePort=0, ipcPort=46245, storageInfo=lv=-57;cid=testClusterID;nsid=991965062;c=1731875704920):Failed to transfer BP-1477976244-172.17.0.2-1731875704920:blk_1073741858_1041 to 127.0.0.1:35683 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:35:34,423 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@45975e5[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:35007, datanodeUuid=96f27179-3395-467d-8cec-ee4e6e5e306e, infoPort=34837, infoSecurePort=0, ipcPort=46245, storageInfo=lv=-57;cid=testClusterID;nsid=991965062;c=1731875704920):Failed to transfer BP-1477976244-172.17.0.2-1731875704920:blk_1073741873_1056 to 127.0.0.1:41145 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:35:35,154 INFO  [regionserver/0b0b0ae8de03:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:35,614 INFO  [regionserver/0b0b0ae8de03:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:35,620 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:36,423 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@66f1b6[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:35007, datanodeUuid=96f27179-3395-467d-8cec-ee4e6e5e306e, infoPort=34837, infoSecurePort=0, ipcPort=46245, storageInfo=lv=-57;cid=testClusterID;nsid=991965062;c=1731875704920):Failed to transfer BP-1477976244-172.17.0.2-1731875704920:blk_1073741888_1071 to 127.0.0.1:35683 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:35:36,428 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33949 is added to blk_1073741883_1066 (size=6027)
2024-11-17T20:35:37,155 INFO  [regionserver/0b0b0ae8de03:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:37,217 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-17T20:35:37,615 INFO  [regionserver/0b0b0ae8de03:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:37,620 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:37,902 ERROR [FSHLog-0-hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData-prefix:0b0b0ae8de03,40559,1731875707247 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:37,902 WARN  [FSHLog-0-hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData-prefix:0b0b0ae8de03,40559,1731875707247 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:37,902 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(198): WAL FSHLog 0b0b0ae8de03%2C40559%2C1731875707247:(num 1731875707650) roll requested
2024-11-17T20:35:37,903 INFO  [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0b0b0ae8de03%2C40559%2C1731875707247.1731875737902
2024-11-17T20:35:37,906 WARN  [Thread-1014 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741889_1072
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:37,906 WARN  [Thread-1014 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1477976244-172.17.0.2-1731875704920:blk_1073741889_1072 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35683,DS-71cd9595-34b3-4e78-8dba-8b432c0955e2,DISK], DatanodeInfoWithStorage[127.0.0.1:33949,DS-59ce2296-6caf-48fd-a174-2d36e11610d2,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35683,DS-71cd9595-34b3-4e78-8dba-8b432c0955e2,DISK]) is bad.
2024-11-17T20:35:37,906 WARN  [Thread-1014 {}] hdfs.DataStreamer(1850): Abandoning BP-1477976244-172.17.0.2-1731875704920:blk_1073741889_1072
2024-11-17T20:35:37,907 WARN  [Thread-1014 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35683,DS-71cd9595-34b3-4e78-8dba-8b432c0955e2,DISK]
2024-11-17T20:35:37,909 WARN  [Thread-1014 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741890_1073
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:37,909 WARN  [Thread-1014 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1477976244-172.17.0.2-1731875704920:blk_1073741890_1073 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38211,DS-c88e28b9-4de5-4077-a5d2-a5f42ad2b57f,DISK], DatanodeInfoWithStorage[127.0.0.1:35007,DS-e3ca24df-f174-47d5-ac27-0b38fd48e050,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38211,DS-c88e28b9-4de5-4077-a5d2-a5f42ad2b57f,DISK]) is bad.
2024-11-17T20:35:37,909 WARN  [Thread-1014 {}] hdfs.DataStreamer(1850): Abandoning BP-1477976244-172.17.0.2-1731875704920:blk_1073741890_1073
2024-11-17T20:35:37,910 WARN  [Thread-1014 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38211,DS-c88e28b9-4de5-4077-a5d2-a5f42ad2b57f,DISK]
2024-11-17T20:35:37,916 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:35:37,916 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:35:37,916 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:35:37,916 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:35:37,916 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:35:37,917 INFO  [master:store-WAL-Roller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650 with entries=54, filesize=26.67 KB; new WAL /user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875737902
2024-11-17T20:35:37,917 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:37,917 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:37,918 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
2024-11-17T20:35:37,918 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36805:36805),(127.0.0.1/127.0.0.1:34837:34837)]
2024-11-17T20:35:37,918 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(879): hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650 is not closed yet, will try archiving it next time
2024-11-17T20:35:37,918 WARN  [IPC Server handler 3 on default port 33551 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650 has not been closed. Lease recovery is in progress. RecoveryId = 1075 for block blk_1073741830_1014
2024-11-17T20:35:37,918 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650 after 0ms
2024-11-17T20:35:39,155 INFO  [regionserver/0b0b0ae8de03:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:39,615 INFO  [regionserver/0b0b0ae8de03:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:41,155 INFO  [regionserver/0b0b0ae8de03:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:41,616 INFO  [regionserver/0b0b0ae8de03:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:41,920 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650 after 4002ms
2024-11-17T20:35:43,156 INFO  [regionserver/0b0b0ae8de03:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:43,616 INFO  [regionserver/0b0b0ae8de03:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:44,389 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@69053978 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1477976244-172.17.0.2-1731875704920:blk_1073741832_1008, datanode=DatanodeInfoWithStorage[127.0.0.1:41145,null,null])
java.net.ConnectException: Call From 0b0b0ae8de03/172.17.0.2 to localhost:42067 failed on connection exception: java.net.ConnectException: Connection refused; For more details see:  http://wiki.apache.org/hadoop/ConnectionRefused
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:876) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:668) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
2024-11-17T20:35:44,392 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33949 is added to blk_1073741832_1020 (size=455)
2024-11-17T20:35:44,568 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.1731875708280 to hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/oldWALs/0b0b0ae8de03%2C33133%2C1731875707434.1731875708280
2024-11-17T20:35:44,570 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.1731875729565 to hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/oldWALs/0b0b0ae8de03%2C33133%2C1731875707434.1731875729565
2024-11-17T20:35:45,156 INFO  [regionserver/0b0b0ae8de03:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:45,364 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@316d3d1d[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:33949, datanodeUuid=6506fff5-03d5-479e-8b59-d4ac07be411d, infoPort=36805, infoSecurePort=0, ipcPort=42875, storageInfo=lv=-57;cid=testClusterID;nsid=991965062;c=1731875704920):Failed to transfer BP-1477976244-172.17.0.2-1731875704920:blk_1073741832_1020 to 127.0.0.1:35683 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:35:45,617 INFO  [regionserver/0b0b0ae8de03:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:47,157 INFO  [regionserver/0b0b0ae8de03:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:47,419 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0b0b0ae8de03%2C33133%2C1731875707434.1731875747418
2024-11-17T20:35:47,423 WARN  [Thread-1022 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741892_1076
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:47,423 WARN  [Thread-1022 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1477976244-172.17.0.2-1731875704920:blk_1073741892_1076 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38211,DS-c88e28b9-4de5-4077-a5d2-a5f42ad2b57f,DISK], DatanodeInfoWithStorage[127.0.0.1:33949,DS-59ce2296-6caf-48fd-a174-2d36e11610d2,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38211,DS-c88e28b9-4de5-4077-a5d2-a5f42ad2b57f,DISK]) is bad.
2024-11-17T20:35:47,423 WARN  [Thread-1022 {}] hdfs.DataStreamer(1850): Abandoning BP-1477976244-172.17.0.2-1731875704920:blk_1073741892_1076
2024-11-17T20:35:47,424 WARN  [Thread-1022 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38211,DS-c88e28b9-4de5-4077-a5d2-a5f42ad2b57f,DISK]
2024-11-17T20:35:47,427 WARN  [Thread-1022 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741893_1077
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:35683
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:47,427 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1228150533_22 at /127.0.0.1:41206 [Receiving block BP-1477976244-172.17.0.2-1731875704920:blk_1073741893_1077] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/cluster_b9ca40e8-4224-d8ab-7466-d1512b745e25/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/cluster_b9ca40e8-4224-d8ab-7466-d1512b745e25/data/data10]'}, localName='127.0.0.1:35007', datanodeUuid='96f27179-3395-467d-8cec-ee4e6e5e306e', xmitsInProgress=0}:Exception transferring block BP-1477976244-172.17.0.2-1731875704920:blk_1073741893_1077 to mirror 127.0.0.1:35683
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:35:47,428 WARN  [Thread-1022 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1477976244-172.17.0.2-1731875704920:blk_1073741893_1077 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35007,DS-e3ca24df-f174-47d5-ac27-0b38fd48e050,DISK], DatanodeInfoWithStorage[127.0.0.1:35683,DS-71cd9595-34b3-4e78-8dba-8b432c0955e2,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:35683,DS-71cd9595-34b3-4e78-8dba-8b432c0955e2,DISK]) is bad.
2024-11-17T20:35:47,428 WARN  [Thread-1022 {}] hdfs.DataStreamer(1850): Abandoning BP-1477976244-172.17.0.2-1731875704920:blk_1073741893_1077
2024-11-17T20:35:47,428 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1228150533_22 at /127.0.0.1:41206 [Receiving block BP-1477976244-172.17.0.2-1731875704920:blk_1073741893_1077] {}] datanode.BlockReceiver(316): Block 1073741893 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-17T20:35:47,428 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1228150533_22 at /127.0.0.1:41206 [Receiving block BP-1477976244-172.17.0.2-1731875704920:blk_1073741893_1077] {}] datanode.DataXceiver(331): 127.0.0.1:35007:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:41206 dst: /127.0.0.1:35007
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:35:47,429 WARN  [Thread-1022 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35683,DS-71cd9595-34b3-4e78-8dba-8b432c0955e2,DISK]
2024-11-17T20:35:47,431 WARN  [Thread-1022 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741894_1078
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:47,431 WARN  [Thread-1022 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1477976244-172.17.0.2-1731875704920:blk_1073741894_1078 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK], DatanodeInfoWithStorage[127.0.0.1:33949,DS-59ce2296-6caf-48fd-a174-2d36e11610d2,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]) is bad.
2024-11-17T20:35:47,431 WARN  [Thread-1022 {}] hdfs.DataStreamer(1850): Abandoning BP-1477976244-172.17.0.2-1731875704920:blk_1073741894_1078
2024-11-17T20:35:47,432 WARN  [Thread-1022 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]
2024-11-17T20:35:47,437 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:35:47,437 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:35:47,437 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:35:47,437 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:35:47,438 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:35:47,438 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.1731875731596 with entries=13, filesize=12.60 KB; new WAL /user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.1731875747418
2024-11-17T20:35:47,439 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36805:36805),(127.0.0.1/127.0.0.1:34837:34837)]
2024-11-17T20:35:47,439 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.1731875731596 is not closed yet, will try archiving it next time
2024-11-17T20:35:47,440 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35007 is added to blk_1073741878_1061 (size=12911)
2024-11-17T20:35:47,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33133 {}] regionserver.HRegion(8855): Flush requested on 6ea106502026e091c4a7f9fa49641279
2024-11-17T20:35:47,445 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ea106502026e091c4a7f9fa49641279 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-11-17T20:35:47,453 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/.tmp/info/001d9418033d49c8b7c7a90dcd4fb41a is 1080, key is row0013/info:/1731875747440/Put/seqid=0
2024-11-17T20:35:47,455 WARN  [Thread-1029 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741896_1080
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:47,455 WARN  [Thread-1029 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1477976244-172.17.0.2-1731875704920:blk_1073741896_1080 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35683,DS-71cd9595-34b3-4e78-8dba-8b432c0955e2,DISK], DatanodeInfoWithStorage[127.0.0.1:33949,DS-59ce2296-6caf-48fd-a174-2d36e11610d2,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35683,DS-71cd9595-34b3-4e78-8dba-8b432c0955e2,DISK]) is bad.
2024-11-17T20:35:47,455 WARN  [Thread-1029 {}] hdfs.DataStreamer(1850): Abandoning BP-1477976244-172.17.0.2-1731875704920:blk_1073741896_1080
2024-11-17T20:35:47,456 WARN  [Thread-1029 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35683,DS-71cd9595-34b3-4e78-8dba-8b432c0955e2,DISK]
2024-11-17T20:35:47,457 WARN  [Thread-1029 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741897_1081
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:47,457 WARN  [Thread-1029 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1477976244-172.17.0.2-1731875704920:blk_1073741897_1081 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38211,DS-c88e28b9-4de5-4077-a5d2-a5f42ad2b57f,DISK], DatanodeInfoWithStorage[127.0.0.1:33949,DS-59ce2296-6caf-48fd-a174-2d36e11610d2,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38211,DS-c88e28b9-4de5-4077-a5d2-a5f42ad2b57f,DISK]) is bad.
2024-11-17T20:35:47,457 WARN  [Thread-1029 {}] hdfs.DataStreamer(1850): Abandoning BP-1477976244-172.17.0.2-1731875704920:blk_1073741897_1081
2024-11-17T20:35:47,458 WARN  [Thread-1029 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38211,DS-c88e28b9-4de5-4077-a5d2-a5f42ad2b57f,DISK]
2024-11-17T20:35:47,466 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33949 is added to blk_1073741898_1082 (size=8190)
2024-11-17T20:35:47,466 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35007 is added to blk_1073741898_1082 (size=8190)
2024-11-17T20:35:47,467 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/.tmp/info/001d9418033d49c8b7c7a90dcd4fb41a
2024-11-17T20:35:47,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/.tmp/info/001d9418033d49c8b7c7a90dcd4fb41a as hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/info/001d9418033d49c8b7c7a90dcd4fb41a
2024-11-17T20:35:47,481 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/info/001d9418033d49c8b7c7a90dcd4fb41a, entries=3, sequenceid=66, filesize=8.0 K
2024-11-17T20:35:47,483 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7527, heapSize ~8.11 KB/8304, currentSize=9.46 KB/9683 for 6ea106502026e091c4a7f9fa49641279 in 37ms, sequenceid=66, compaction requested=false
2024-11-17T20:35:47,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ea106502026e091c4a7f9fa49641279:

2024-11-17T20:35:47,483 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=25.7 K, sizeToCheck=16.0 K
2024-11-17T20:35:47,483 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-17T20:35:47,483 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/info/6f1900438fd24cc191be2c8da16b4e9e because midkey is the same as first or last row
2024-11-17T20:35:47,617 INFO  [regionserver/0b0b0ae8de03:0.logRoller {}] wal.FSHLog(556): LowReplication-Roller was enabled.
2024-11-17T20:35:47,617 INFO  [regionserver/0b0b0ae8de03:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:47,669 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-17T20:35:47,669 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-17T20:35:47,670 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-17T20:35:47,670 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T20:35:47,670 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T20:35:47,670 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-17T20:35:47,670 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-17T20:35:47,670 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=627172387, stopped=false
2024-11-17T20:35:47,670 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=0b0b0ae8de03,40559,1731875707247
2024-11-17T20:35:47,728 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33133-0x1014a8214ff0001, quorum=127.0.0.1:54994, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-17T20:35:47,728 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37509-0x1014a8214ff0002, quorum=127.0.0.1:54994, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-17T20:35:47,728 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40559-0x1014a8214ff0000, quorum=127.0.0.1:54994, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-17T20:35:47,728 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37509-0x1014a8214ff0002, quorum=127.0.0.1:54994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:35:47,728 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33133-0x1014a8214ff0001, quorum=127.0.0.1:54994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:35:47,728 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40559-0x1014a8214ff0000, quorum=127.0.0.1:54994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:35:47,728 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-17T20:35:47,729 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-17T20:35:47,729 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-17T20:35:47,729 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T20:35:47,730 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33133-0x1014a8214ff0001, quorum=127.0.0.1:54994, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-17T20:35:47,730 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '0b0b0ae8de03,33133,1731875707434' *****
2024-11-17T20:35:47,730 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-17T20:35:47,730 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37509-0x1014a8214ff0002, quorum=127.0.0.1:54994, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-17T20:35:47,730 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:40559-0x1014a8214ff0000, quorum=127.0.0.1:54994, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-17T20:35:47,730 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '0b0b0ae8de03,37509,1731875709020' *****
2024-11-17T20:35:47,730 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-17T20:35:47,730 INFO  [RS:0;0b0b0ae8de03:33133 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-17T20:35:47,730 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-17T20:35:47,730 INFO  [RS:0;0b0b0ae8de03:33133 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-17T20:35:47,731 INFO  [RS:1;0b0b0ae8de03:37509 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-17T20:35:47,731 INFO  [RS:0;0b0b0ae8de03:33133 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-17T20:35:47,731 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-17T20:35:47,731 INFO  [RS:1;0b0b0ae8de03:37509 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-17T20:35:47,731 INFO  [RS:0;0b0b0ae8de03:33133 {}] regionserver.HRegionServer(3091): Received CLOSE for 6ea106502026e091c4a7f9fa49641279
2024-11-17T20:35:47,731 INFO  [RS:1;0b0b0ae8de03:37509 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-17T20:35:47,731 INFO  [RS:1;0b0b0ae8de03:37509 {}] regionserver.HRegionServer(959): stopping server 0b0b0ae8de03,37509,1731875709020
2024-11-17T20:35:47,731 INFO  [RS:1;0b0b0ae8de03:37509 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-17T20:35:47,732 INFO  [RS:1;0b0b0ae8de03:37509 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;0b0b0ae8de03:37509.
2024-11-17T20:35:47,732 INFO  [RS:0;0b0b0ae8de03:33133 {}] regionserver.HRegionServer(959): stopping server 0b0b0ae8de03,33133,1731875707434
2024-11-17T20:35:47,732 DEBUG [RS:1;0b0b0ae8de03:37509 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-17T20:35:47,732 INFO  [RS:0;0b0b0ae8de03:33133 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-17T20:35:47,732 DEBUG [RS:1;0b0b0ae8de03:37509 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T20:35:47,732 INFO  [RS:0;0b0b0ae8de03:33133 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;0b0b0ae8de03:33133.
2024-11-17T20:35:47,732 INFO  [RS:1;0b0b0ae8de03:37509 {}] regionserver.HRegionServer(976): stopping server 0b0b0ae8de03,37509,1731875709020; all regions closed.
2024-11-17T20:35:47,732 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6ea106502026e091c4a7f9fa49641279, disabling compactions & flushes
2024-11-17T20:35:47,732 DEBUG [RS:0;0b0b0ae8de03:33133 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-17T20:35:47,732 INFO  [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1731875709192.6ea106502026e091c4a7f9fa49641279.
2024-11-17T20:35:47,732 DEBUG [RS:0;0b0b0ae8de03:33133 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T20:35:47,732 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1731875709192.6ea106502026e091c4a7f9fa49641279.
2024-11-17T20:35:47,732 INFO  [RS:0;0b0b0ae8de03:33133 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-17T20:35:47,732 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1731875709192.6ea106502026e091c4a7f9fa49641279. after waiting 0 ms
2024-11-17T20:35:47,732 INFO  [RS:0;0b0b0ae8de03:33133 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-17T20:35:47,732 INFO  [RS:0;0b0b0ae8de03:33133 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-17T20:35:47,732 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1731875709192.6ea106502026e091c4a7f9fa49641279.
2024-11-17T20:35:47,732 INFO  [RS:0;0b0b0ae8de03:33133 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-17T20:35:47,733 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:35:47,733 INFO  [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 6ea106502026e091c4a7f9fa49641279 1/1 column families, dataSize=9.46 KB heapSize=10.38 KB
2024-11-17T20:35:47,733 INFO  [RS:0;0b0b0ae8de03:33133 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-11-17T20:35:47,733 DEBUG [RS:0;0b0b0ae8de03:33133 {}] regionserver.HRegionServer(1325): Online Regions={6ea106502026e091c4a7f9fa49641279=TestLogRolling-testLogRollOnDatanodeDeath,,1731875709192.6ea106502026e091c4a7f9fa49641279., 1588230740=hbase:meta,,1.1588230740}
2024-11-17T20:35:47,733 DEBUG [RS:0;0b0b0ae8de03:33133 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 6ea106502026e091c4a7f9fa49641279
2024-11-17T20:35:47,733 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-17T20:35:47,733 INFO  [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-17T20:35:47,733 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-17T20:35:47,733 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-17T20:35:47,733 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-17T20:35:47,734 INFO  [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.71 KB heapSize=3.75 KB
2024-11-17T20:35:47,734 ERROR [FSHLog-0-hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899-prefix:0b0b0ae8de03,33133,1731875707434.meta {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:47,734 WARN  [FSHLog-0-hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899-prefix:0b0b0ae8de03,33133,1731875707434.meta {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:47,734 DEBUG [regionserver/0b0b0ae8de03:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 0b0b0ae8de03%2C33133%2C1731875707434.meta:.meta(num 1731875708822) roll requested
2024-11-17T20:35:47,735 INFO  [regionserver/0b0b0ae8de03:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875747734.meta
2024-11-17T20:35:47,740 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:35:47,740 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:35:47,740 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:35:47,740 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:35:47,743 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:47,743 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:47,744 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
2024-11-17T20:35:47,744 WARN  [Thread-1037 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741899_1083
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:35683
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:47,744 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-723733033_22 at /127.0.0.1:43044 [Receiving block BP-1477976244-172.17.0.2-1731875704920:blk_1073741899_1083] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/cluster_b9ca40e8-4224-d8ab-7466-d1512b745e25/data/data3, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/cluster_b9ca40e8-4224-d8ab-7466-d1512b745e25/data/data4]'}, localName='127.0.0.1:33949', datanodeUuid='6506fff5-03d5-479e-8b59-d4ac07be411d', xmitsInProgress=0}:Exception transferring block BP-1477976244-172.17.0.2-1731875704920:blk_1073741899_1083 to mirror 127.0.0.1:35683
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:35:47,744 WARN  [Thread-1037 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1477976244-172.17.0.2-1731875704920:blk_1073741899_1083 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33949,DS-59ce2296-6caf-48fd-a174-2d36e11610d2,DISK], DatanodeInfoWithStorage[127.0.0.1:35683,DS-71cd9595-34b3-4e78-8dba-8b432c0955e2,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:35683,DS-71cd9595-34b3-4e78-8dba-8b432c0955e2,DISK]) is bad.
2024-11-17T20:35:47,744 WARN  [Thread-1037 {}] hdfs.DataStreamer(1850): Abandoning BP-1477976244-172.17.0.2-1731875704920:blk_1073741899_1083
2024-11-17T20:35:47,744 WARN  [IPC Server handler 0 on default port 33551 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295 has not been closed. Lease recovery is in progress. RecoveryId = 1084 for block blk_1073741836_1012
2024-11-17T20:35:47,744 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-723733033_22 at /127.0.0.1:43044 [Receiving block BP-1477976244-172.17.0.2-1731875704920:blk_1073741899_1083] {}] datanode.BlockReceiver(316): Block 1073741899 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-17T20:35:47,744 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-723733033_22 at /127.0.0.1:43044 [Receiving block BP-1477976244-172.17.0.2-1731875704920:blk_1073741899_1083] {}] datanode.DataXceiver(331): 127.0.0.1:33949:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:43044 dst: /127.0.0.1:33949
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:35:47,745 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295 after 1ms
2024-11-17T20:35:47,745 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/.tmp/info/1dc69b9b2c9543f6a894f57b428a84d1 is 1080, key is row0015/info:/1731875747447/Put/seqid=0
2024-11-17T20:35:47,746 WARN  [Thread-1037 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35683,DS-71cd9595-34b3-4e78-8dba-8b432c0955e2,DISK]
2024-11-17T20:35:47,747 WARN  [Thread-1038 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741900_1085
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:47,747 WARN  [Thread-1038 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1477976244-172.17.0.2-1731875704920:blk_1073741900_1085 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK], DatanodeInfoWithStorage[127.0.0.1:38211,DS-c88e28b9-4de5-4077-a5d2-a5f42ad2b57f,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]) is bad.
2024-11-17T20:35:47,747 WARN  [Thread-1038 {}] hdfs.DataStreamer(1850): Abandoning BP-1477976244-172.17.0.2-1731875704920:blk_1073741900_1085
2024-11-17T20:35:47,748 WARN  [Thread-1038 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]
2024-11-17T20:35:47,748 WARN  [Thread-1037 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741901_1086
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:38211
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:47,748 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-723733033_22 at /127.0.0.1:41244 [Receiving block BP-1477976244-172.17.0.2-1731875704920:blk_1073741901_1086] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/cluster_b9ca40e8-4224-d8ab-7466-d1512b745e25/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/cluster_b9ca40e8-4224-d8ab-7466-d1512b745e25/data/data10]'}, localName='127.0.0.1:35007', datanodeUuid='96f27179-3395-467d-8cec-ee4e6e5e306e', xmitsInProgress=0}:Exception transferring block BP-1477976244-172.17.0.2-1731875704920:blk_1073741901_1086 to mirror 127.0.0.1:38211
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:35:47,749 WARN  [Thread-1037 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1477976244-172.17.0.2-1731875704920:blk_1073741901_1086 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35007,DS-e3ca24df-f174-47d5-ac27-0b38fd48e050,DISK], DatanodeInfoWithStorage[127.0.0.1:38211,DS-c88e28b9-4de5-4077-a5d2-a5f42ad2b57f,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:38211,DS-c88e28b9-4de5-4077-a5d2-a5f42ad2b57f,DISK]) is bad.
2024-11-17T20:35:47,749 WARN  [Thread-1037 {}] hdfs.DataStreamer(1850): Abandoning BP-1477976244-172.17.0.2-1731875704920:blk_1073741901_1086
2024-11-17T20:35:47,749 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-723733033_22 at /127.0.0.1:41244 [Receiving block BP-1477976244-172.17.0.2-1731875704920:blk_1073741901_1086] {}] datanode.BlockReceiver(316): Block 1073741901 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-17T20:35:47,749 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-723733033_22 at /127.0.0.1:41244 [Receiving block BP-1477976244-172.17.0.2-1731875704920:blk_1073741901_1086] {}] datanode.DataXceiver(331): 127.0.0.1:35007:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:41244 dst: /127.0.0.1:35007
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:35:47,750 WARN  [Thread-1037 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38211,DS-c88e28b9-4de5-4077-a5d2-a5f42ad2b57f,DISK]
2024-11-17T20:35:47,751 WARN  [Thread-1038 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741902_1087
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:38211
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:47,751 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-723733033_22 at /127.0.0.1:41254 [Receiving block BP-1477976244-172.17.0.2-1731875704920:blk_1073741902_1087] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/cluster_b9ca40e8-4224-d8ab-7466-d1512b745e25/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/cluster_b9ca40e8-4224-d8ab-7466-d1512b745e25/data/data10]'}, localName='127.0.0.1:35007', datanodeUuid='96f27179-3395-467d-8cec-ee4e6e5e306e', xmitsInProgress=0}:Exception transferring block BP-1477976244-172.17.0.2-1731875704920:blk_1073741902_1087 to mirror 127.0.0.1:38211
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:35:47,751 WARN  [Thread-1038 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1477976244-172.17.0.2-1731875704920:blk_1073741902_1087 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35007,DS-e3ca24df-f174-47d5-ac27-0b38fd48e050,DISK], DatanodeInfoWithStorage[127.0.0.1:38211,DS-c88e28b9-4de5-4077-a5d2-a5f42ad2b57f,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:38211,DS-c88e28b9-4de5-4077-a5d2-a5f42ad2b57f,DISK]) is bad.
2024-11-17T20:35:47,751 WARN  [Thread-1038 {}] hdfs.DataStreamer(1850): Abandoning BP-1477976244-172.17.0.2-1731875704920:blk_1073741902_1087
2024-11-17T20:35:47,751 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-723733033_22 at /127.0.0.1:41254 [Receiving block BP-1477976244-172.17.0.2-1731875704920:blk_1073741902_1087] {}] datanode.BlockReceiver(316): Block 1073741902 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-17T20:35:47,751 WARN  [Thread-1037 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741903_1088
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:47,751 WARN  [Thread-1037 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1477976244-172.17.0.2-1731875704920:blk_1073741903_1088 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK], DatanodeInfoWithStorage[127.0.0.1:33949,DS-59ce2296-6caf-48fd-a174-2d36e11610d2,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]) is bad.
2024-11-17T20:35:47,751 WARN  [Thread-1037 {}] hdfs.DataStreamer(1850): Abandoning BP-1477976244-172.17.0.2-1731875704920:blk_1073741903_1088
2024-11-17T20:35:47,751 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-723733033_22 at /127.0.0.1:41254 [Receiving block BP-1477976244-172.17.0.2-1731875704920:blk_1073741902_1087] {}] datanode.DataXceiver(331): 127.0.0.1:35007:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:41254 dst: /127.0.0.1:35007
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:35:47,752 WARN  [Thread-1038 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38211,DS-c88e28b9-4de5-4077-a5d2-a5f42ad2b57f,DISK]
2024-11-17T20:35:47,752 WARN  [Thread-1037 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]
2024-11-17T20:35:47,753 WARN  [Thread-1038 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741904_1089
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:47,753 WARN  [Thread-1038 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1477976244-172.17.0.2-1731875704920:blk_1073741904_1089 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35683,DS-71cd9595-34b3-4e78-8dba-8b432c0955e2,DISK], DatanodeInfoWithStorage[127.0.0.1:35007,DS-e3ca24df-f174-47d5-ac27-0b38fd48e050,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35683,DS-71cd9595-34b3-4e78-8dba-8b432c0955e2,DISK]) is bad.
2024-11-17T20:35:47,753 WARN  [Thread-1038 {}] hdfs.DataStreamer(1850): Abandoning BP-1477976244-172.17.0.2-1731875704920:blk_1073741904_1089
2024-11-17T20:35:47,754 WARN  [Thread-1038 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35683,DS-71cd9595-34b3-4e78-8dba-8b432c0955e2,DISK]
2024-11-17T20:35:47,762 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:35:47,762 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:35:47,763 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:35:47,763 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:35:47,763 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:35:47,763 INFO  [regionserver/0b0b0ae8de03:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta with entries=8, filesize=2.33 KB; new WAL /user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875747734.meta
2024-11-17T20:35:47,764 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35007 is added to blk_1073741906_1091 (size=14660)
2024-11-17T20:35:47,764 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33949 is added to blk_1073741906_1091 (size=14660)
2024-11-17T20:35:47,765 INFO  [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=9.46 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/.tmp/info/1dc69b9b2c9543f6a894f57b428a84d1
2024-11-17T20:35:47,768 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:47,768 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:47,769 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
2024-11-17T20:35:47,769 WARN  [IPC Server handler 4 on default port 33551 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta has not been closed. Lease recovery is in progress. RecoveryId = 1092 for block blk_1073741834_1010
2024-11-17T20:35:47,770 DEBUG [regionserver/0b0b0ae8de03:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36805:36805),(127.0.0.1/127.0.0.1:34837:34837)]
2024-11-17T20:35:47,770 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta after 1ms
2024-11-17T20:35:47,770 DEBUG [regionserver/0b0b0ae8de03:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta is not closed yet, will try archiving it next time
2024-11-17T20:35:47,773 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/.tmp/info/1dc69b9b2c9543f6a894f57b428a84d1 as hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/info/1dc69b9b2c9543f6a894f57b428a84d1
2024-11-17T20:35:47,780 INFO  [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/info/1dc69b9b2c9543f6a894f57b428a84d1, entries=9, sequenceid=78, filesize=14.3 K
2024-11-17T20:35:47,781 INFO  [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~9.46 KB/9683, heapSize ~10.36 KB/10608, currentSize=0 B/0 for 6ea106502026e091c4a7f9fa49641279 in 49ms, sequenceid=78, compaction requested=true
2024-11-17T20:35:47,782 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731875709192.6ea106502026e091c4a7f9fa49641279.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/info/3f1fe0c9aab24b899b5dfe69f3f5857c, hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/info/a633fbb8c8994446b4e51b16e3c0a02d, hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/info/234c08f58e3642d895af9af2ea82de0a, hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/info/3071f7c88b2e44749b1d0c1483a5c9f2, hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/info/1f46bd98a1db4ac1aa38719addc502bf, hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/info/290181b0ca71497bbc71c5f1dc48c46a] to archive
2024-11-17T20:35:47,783 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731875709192.6ea106502026e091c4a7f9fa49641279.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-17T20:35:47,785 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731875709192.6ea106502026e091c4a7f9fa49641279.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/info/3f1fe0c9aab24b899b5dfe69f3f5857c to hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/info/3f1fe0c9aab24b899b5dfe69f3f5857c
2024-11-17T20:35:47,787 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731875709192.6ea106502026e091c4a7f9fa49641279.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/info/a633fbb8c8994446b4e51b16e3c0a02d to hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/info/a633fbb8c8994446b4e51b16e3c0a02d
2024-11-17T20:35:47,789 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731875709192.6ea106502026e091c4a7f9fa49641279.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/info/234c08f58e3642d895af9af2ea82de0a to hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/info/234c08f58e3642d895af9af2ea82de0a
2024-11-17T20:35:47,789 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/hbase/meta/1588230740/.tmp/info/b154b019484441389b57db5f7b367e6b is 203, key is TestLogRolling-testLogRollOnDatanodeDeath,,1731875709192.6ea106502026e091c4a7f9fa49641279./info:regioninfo/1731875709995/Put/seqid=0
2024-11-17T20:35:47,791 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731875709192.6ea106502026e091c4a7f9fa49641279.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/info/3071f7c88b2e44749b1d0c1483a5c9f2 to hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/info/3071f7c88b2e44749b1d0c1483a5c9f2
2024-11-17T20:35:47,791 WARN  [Thread-1052 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741907_1093
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:47,791 WARN  [Thread-1052 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1477976244-172.17.0.2-1731875704920:blk_1073741907_1093 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35683,DS-71cd9595-34b3-4e78-8dba-8b432c0955e2,DISK], DatanodeInfoWithStorage[127.0.0.1:38211,DS-c88e28b9-4de5-4077-a5d2-a5f42ad2b57f,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35683,DS-71cd9595-34b3-4e78-8dba-8b432c0955e2,DISK]) is bad.
2024-11-17T20:35:47,791 WARN  [Thread-1052 {}] hdfs.DataStreamer(1850): Abandoning BP-1477976244-172.17.0.2-1731875704920:blk_1073741907_1093
2024-11-17T20:35:47,792 WARN  [Thread-1052 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35683,DS-71cd9595-34b3-4e78-8dba-8b432c0955e2,DISK]
2024-11-17T20:35:47,793 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731875709192.6ea106502026e091c4a7f9fa49641279.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/info/1f46bd98a1db4ac1aa38719addc502bf to hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/info/1f46bd98a1db4ac1aa38719addc502bf
2024-11-17T20:35:47,793 WARN  [Thread-1052 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741908_1094
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:47,794 WARN  [Thread-1052 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1477976244-172.17.0.2-1731875704920:blk_1073741908_1094 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38211,DS-c88e28b9-4de5-4077-a5d2-a5f42ad2b57f,DISK], DatanodeInfoWithStorage[127.0.0.1:35007,DS-e3ca24df-f174-47d5-ac27-0b38fd48e050,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38211,DS-c88e28b9-4de5-4077-a5d2-a5f42ad2b57f,DISK]) is bad.
2024-11-17T20:35:47,794 WARN  [Thread-1052 {}] hdfs.DataStreamer(1850): Abandoning BP-1477976244-172.17.0.2-1731875704920:blk_1073741908_1094
2024-11-17T20:35:47,794 WARN  [Thread-1052 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38211,DS-c88e28b9-4de5-4077-a5d2-a5f42ad2b57f,DISK]
2024-11-17T20:35:47,794 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731875709192.6ea106502026e091c4a7f9fa49641279.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/info/290181b0ca71497bbc71c5f1dc48c46a to hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/info/290181b0ca71497bbc71c5f1dc48c46a
2024-11-17T20:35:47,795 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731875709192.6ea106502026e091c4a7f9fa49641279.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=0b0b0ae8de03:40559 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 16 more
2024-11-17T20:35:47,796 WARN  [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731875709192.6ea106502026e091c4a7f9fa49641279.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [3f1fe0c9aab24b899b5dfe69f3f5857c=10347, a633fbb8c8994446b4e51b16e3c0a02d=12506, 234c08f58e3642d895af9af2ea82de0a=17994, 3071f7c88b2e44749b1d0c1483a5c9f2=6027, 1f46bd98a1db4ac1aa38719addc502bf=6027, 290181b0ca71497bbc71c5f1dc48c46a=6027]
2024-11-17T20:35:47,799 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33949 is added to blk_1073741909_1095 (size=7089)
2024-11-17T20:35:47,800 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35007 is added to blk_1073741909_1095 (size=7089)
2024-11-17T20:35:47,800 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6ea106502026e091c4a7f9fa49641279/recovered.edits/81.seqid, newMaxSeqId=81, maxSeqId=1
2024-11-17T20:35:47,801 INFO  [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.50 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/hbase/meta/1588230740/.tmp/info/b154b019484441389b57db5f7b367e6b
2024-11-17T20:35:47,801 INFO  [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1731875709192.6ea106502026e091c4a7f9fa49641279.
2024-11-17T20:35:47,801 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6ea106502026e091c4a7f9fa49641279:
Waiting for close lock at 1731875747732Running coprocessor pre-close hooks at 1731875747732Disabling compacts and flushes for region at 1731875747732Disabling writes for close at 1731875747732Obtaining lock to block concurrent updates at 1731875747733 (+1 ms)Preparing flush snapshotting stores in 6ea106502026e091c4a7f9fa49641279 at 1731875747733Finished memstore snapshotting TestLogRolling-testLogRollOnDatanodeDeath,,1731875709192.6ea106502026e091c4a7f9fa49641279., syncing WAL and waiting on mvcc, flushsize=dataSize=9683, getHeapSize=10608, getOffHeapSize=0, getCellsCount=9 at 1731875747733Flushing stores of TestLogRolling-testLogRollOnDatanodeDeath,,1731875709192.6ea106502026e091c4a7f9fa49641279. at 1731875747734 (+1 ms)Flushing 6ea106502026e091c4a7f9fa49641279/info: creating writer at 1731875747734Flushing 6ea106502026e091c4a7f9fa49641279/info: appending metadata at 1731875747745 (+11 ms)Flushing 6ea106502026e091c4a7f9fa49641279/info: closing flushed file at 1731875747745Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@14522ddd: reopening flushed file at 1731875747772 (+27 ms)Finished flush of dataSize ~9.46 KB/9683, heapSize ~10.36 KB/10608, currentSize=0 B/0 for 6ea106502026e091c4a7f9fa49641279 in 49ms, sequenceid=78, compaction requested=true at 1731875747781 (+9 ms)Writing region close event to WAL at 1731875747796 (+15 ms)Running coprocessor post-close hooks at 1731875747801 (+5 ms)Closed at 1731875747801
2024-11-17T20:35:47,801 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1731875709192.6ea106502026e091c4a7f9fa49641279.
2024-11-17T20:35:47,825 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/hbase/meta/1588230740/.tmp/ns/f6da818bd77848a697d532029e82d146 is 43, key is default/ns:d/1731875708943/Put/seqid=0
2024-11-17T20:35:47,828 WARN  [Thread-1059 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741910_1096
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:35683
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:47,828 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-723733033_22 at /127.0.0.1:41296 [Receiving block BP-1477976244-172.17.0.2-1731875704920:blk_1073741910_1096] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/cluster_b9ca40e8-4224-d8ab-7466-d1512b745e25/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/cluster_b9ca40e8-4224-d8ab-7466-d1512b745e25/data/data10]'}, localName='127.0.0.1:35007', datanodeUuid='96f27179-3395-467d-8cec-ee4e6e5e306e', xmitsInProgress=0}:Exception transferring block BP-1477976244-172.17.0.2-1731875704920:blk_1073741910_1096 to mirror 127.0.0.1:35683
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:35:47,828 WARN  [Thread-1059 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1477976244-172.17.0.2-1731875704920:blk_1073741910_1096 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35007,DS-e3ca24df-f174-47d5-ac27-0b38fd48e050,DISK], DatanodeInfoWithStorage[127.0.0.1:35683,DS-71cd9595-34b3-4e78-8dba-8b432c0955e2,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:35683,DS-71cd9595-34b3-4e78-8dba-8b432c0955e2,DISK]) is bad.
2024-11-17T20:35:47,828 WARN  [Thread-1059 {}] hdfs.DataStreamer(1850): Abandoning BP-1477976244-172.17.0.2-1731875704920:blk_1073741910_1096
2024-11-17T20:35:47,828 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-723733033_22 at /127.0.0.1:41296 [Receiving block BP-1477976244-172.17.0.2-1731875704920:blk_1073741910_1096] {}] datanode.BlockReceiver(316): Block 1073741910 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-17T20:35:47,829 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-723733033_22 at /127.0.0.1:41296 [Receiving block BP-1477976244-172.17.0.2-1731875704920:blk_1073741910_1096] {}] datanode.DataXceiver(331): 127.0.0.1:35007:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:41296 dst: /127.0.0.1:35007
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:35:47,829 WARN  [Thread-1059 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35683,DS-71cd9595-34b3-4e78-8dba-8b432c0955e2,DISK]
2024-11-17T20:35:47,830 WARN  [Thread-1059 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741911_1097
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:47,830 WARN  [Thread-1059 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1477976244-172.17.0.2-1731875704920:blk_1073741911_1097 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38211,DS-c88e28b9-4de5-4077-a5d2-a5f42ad2b57f,DISK], DatanodeInfoWithStorage[127.0.0.1:33949,DS-59ce2296-6caf-48fd-a174-2d36e11610d2,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38211,DS-c88e28b9-4de5-4077-a5d2-a5f42ad2b57f,DISK]) is bad.
2024-11-17T20:35:47,830 WARN  [Thread-1059 {}] hdfs.DataStreamer(1850): Abandoning BP-1477976244-172.17.0.2-1731875704920:blk_1073741911_1097
2024-11-17T20:35:47,831 WARN  [Thread-1059 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38211,DS-c88e28b9-4de5-4077-a5d2-a5f42ad2b57f,DISK]
2024-11-17T20:35:47,836 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35007 is added to blk_1073741912_1098 (size=5153)
2024-11-17T20:35:47,836 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33949 is added to blk_1073741912_1098 (size=5153)
2024-11-17T20:35:47,837 INFO  [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/hbase/meta/1588230740/.tmp/ns/f6da818bd77848a697d532029e82d146
2024-11-17T20:35:47,840 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.1731875731596 to hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/oldWALs/0b0b0ae8de03%2C33133%2C1731875707434.1731875731596
2024-11-17T20:35:47,861 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/hbase/meta/1588230740/.tmp/table/42da2f3d69bb440bb96391d31650950c is 77, key is TestLogRolling-testLogRollOnDatanodeDeath/table:state/1731875710008/Put/seqid=0
2024-11-17T20:35:47,863 WARN  [Thread-1066 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741913_1099
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:47,863 WARN  [Thread-1066 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1477976244-172.17.0.2-1731875704920:blk_1073741913_1099 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK], DatanodeInfoWithStorage[127.0.0.1:35007,DS-e3ca24df-f174-47d5-ac27-0b38fd48e050,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]) is bad.
2024-11-17T20:35:47,863 WARN  [Thread-1066 {}] hdfs.DataStreamer(1850): Abandoning BP-1477976244-172.17.0.2-1731875704920:blk_1073741913_1099
2024-11-17T20:35:47,864 WARN  [Thread-1066 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41145,DS-5470270f-a72f-4479-b477-b6d1bcd87291,DISK]
2024-11-17T20:35:47,865 WARN  [Thread-1066 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741914_1100
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:35:47,866 WARN  [Thread-1066 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1477976244-172.17.0.2-1731875704920:blk_1073741914_1100 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35683,DS-71cd9595-34b3-4e78-8dba-8b432c0955e2,DISK], DatanodeInfoWithStorage[127.0.0.1:33949,DS-59ce2296-6caf-48fd-a174-2d36e11610d2,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35683,DS-71cd9595-34b3-4e78-8dba-8b432c0955e2,DISK]) is bad.
2024-11-17T20:35:47,866 WARN  [Thread-1066 {}] hdfs.DataStreamer(1850): Abandoning BP-1477976244-172.17.0.2-1731875704920:blk_1073741914_1100
2024-11-17T20:35:47,866 WARN  [Thread-1066 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35683,DS-71cd9595-34b3-4e78-8dba-8b432c0955e2,DISK]
2024-11-17T20:35:47,871 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35007 is added to blk_1073741915_1101 (size=5424)
2024-11-17T20:35:47,871 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33949 is added to blk_1073741915_1101 (size=5424)
2024-11-17T20:35:47,872 INFO  [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=146 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/hbase/meta/1588230740/.tmp/table/42da2f3d69bb440bb96391d31650950c
2024-11-17T20:35:47,879 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/hbase/meta/1588230740/.tmp/info/b154b019484441389b57db5f7b367e6b as hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/hbase/meta/1588230740/info/b154b019484441389b57db5f7b367e6b
2024-11-17T20:35:47,887 INFO  [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/hbase/meta/1588230740/info/b154b019484441389b57db5f7b367e6b, entries=10, sequenceid=11, filesize=6.9 K
2024-11-17T20:35:47,888 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/hbase/meta/1588230740/.tmp/ns/f6da818bd77848a697d532029e82d146 as hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/hbase/meta/1588230740/ns/f6da818bd77848a697d532029e82d146
2024-11-17T20:35:47,896 INFO  [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/hbase/meta/1588230740/ns/f6da818bd77848a697d532029e82d146, entries=2, sequenceid=11, filesize=5.0 K
2024-11-17T20:35:47,897 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/hbase/meta/1588230740/.tmp/table/42da2f3d69bb440bb96391d31650950c as hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/hbase/meta/1588230740/table/42da2f3d69bb440bb96391d31650950c
2024-11-17T20:35:47,904 INFO  [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/hbase/meta/1588230740/table/42da2f3d69bb440bb96391d31650950c, entries=2, sequenceid=11, filesize=5.3 K
2024-11-17T20:35:47,905 INFO  [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.71 KB/1752, heapSize ~3.45 KB/3536, currentSize=0 B/0 for 1588230740 in 172ms, sequenceid=11, compaction requested=false
2024-11-17T20:35:47,910 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-11-17T20:35:47,911 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-17T20:35:47,911 INFO  [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-17T20:35:47,911 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731875747733Running coprocessor pre-close hooks at 1731875747733Disabling compacts and flushes for region at 1731875747733Disabling writes for close at 1731875747733Obtaining lock to block concurrent updates at 1731875747734 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1731875747734Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1752, getHeapSize=3776, getOffHeapSize=0, getCellsCount=14 at 1731875747734Flushing stores of hbase:meta,,1.1588230740 at 1731875747770 (+36 ms)Flushing 1588230740/info: creating writer at 1731875747770Flushing 1588230740/info: appending metadata at 1731875747789 (+19 ms)Flushing 1588230740/info: closing flushed file at 1731875747789Flushing 1588230740/ns: creating writer at 1731875747807 (+18 ms)Flushing 1588230740/ns: appending metadata at 1731875747825 (+18 ms)Flushing 1588230740/ns: closing flushed file at 1731875747825Flushing 1588230740/table: creating writer at 1731875747844 (+19 ms)Flushing 1588230740/table: appending metadata at 1731875747860 (+16 ms)Flushing 1588230740/table: closing flushed file at 1731875747860Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@29697db3: reopening flushed file at 1731875747878 (+18 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1561ade7: reopening flushed file at 1731875747887 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5f076736: reopening flushed file at 1731875747896 (+9 ms)Finished flush of dataSize ~1.71 KB/1752, heapSize ~3.45 KB/3536, currentSize=0 B/0 for 1588230740 in 172ms, sequenceid=11, compaction requested=false at 1731875747905 (+9 ms)Writing region close event to WAL at 1731875747906 (+1 ms)Running coprocessor post-close hooks at 1731875747911 (+5 ms)Closed at 1731875747911
2024-11-17T20:35:47,911 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-17T20:35:47,933 INFO  [RS:0;0b0b0ae8de03:33133 {}] regionserver.HRegionServer(976): stopping server 0b0b0ae8de03,33133,1731875707434; all regions closed.
2024-11-17T20:35:47,934 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:35:47,934 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:35:47,934 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:35:47,934 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:35:47,934 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:35:47,936 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35007 is added to blk_1073741905_1090 (size=825)
2024-11-17T20:35:47,936 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33949 is added to blk_1073741905_1090 (size=825)
2024-11-17T20:35:48,131 INFO  [regionserver/0b0b0ae8de03:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-11-17T20:35:48,131 INFO  [regionserver/0b0b0ae8de03:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-17T20:35:48,131 INFO  [regionserver/0b0b0ae8de03:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-11-17T20:35:48,153 INFO  [regionserver/0b0b0ae8de03:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-11-17T20:35:48,153 INFO  [regionserver/0b0b0ae8de03:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-11-17T20:35:48,422 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@45975e5[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:35007, datanodeUuid=96f27179-3395-467d-8cec-ee4e6e5e306e, infoPort=34837, infoSecurePort=0, ipcPort=46245, storageInfo=lv=-57;cid=testClusterID;nsid=991965062;c=1731875704920):Failed to transfer BP-1477976244-172.17.0.2-1731875704920:blk_1073741878_1061 to 127.0.0.1:35683 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:35:49,154 INFO  [regionserver/0b0b0ae8de03:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-17T20:35:49,323 INFO  [master/0b0b0ae8de03:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-11-17T20:35:49,323 INFO  [master/0b0b0ae8de03:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-11-17T20:35:51,368 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35007 is added to blk_1073741826_1002 (size=42)
2024-11-17T20:35:51,368 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35007 is added to blk_1073741828_1004 (size=1189)
2024-11-17T20:35:51,746 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295 after 4002ms
2024-11-17T20:35:51,771 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta after 4002ms
2024-11-17T20:35:52,368 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35007 is added to blk_1073741835_1011 (size=393)
2024-11-17T20:35:52,368 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35007 is added to blk_1073741837_1013 (size=76)
2024-11-17T20:35:52,744 ERROR [WAL-Shutdown-0 {}] wal.AbstractFSWAL(2118): We have waited 5 seconds but the close of async writer doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.fshlog.wait.on.shutdown.seconds"
2024-11-17T20:35:52,747 DEBUG [RS:1;0b0b0ae8de03:37509 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/oldWALs
2024-11-17T20:35:52,747 INFO  [RS:1;0b0b0ae8de03:37509 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 0b0b0ae8de03%2C37509%2C1731875709020:(num 1731875709295)
2024-11-17T20:35:52,747 DEBUG [RS:1;0b0b0ae8de03:37509 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T20:35:52,747 INFO  [RS:1;0b0b0ae8de03:37509 {}] regionserver.LeaseManager(133): Closed leases
2024-11-17T20:35:52,747 INFO  [RS:1;0b0b0ae8de03:37509 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-17T20:35:52,748 INFO  [RS:1;0b0b0ae8de03:37509 {}] hbase.ChoreService(370): Chore service for: regionserver/0b0b0ae8de03:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-17T20:35:52,748 INFO  [RS:1;0b0b0ae8de03:37509 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-17T20:35:52,748 INFO  [regionserver/0b0b0ae8de03:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-17T20:35:52,748 INFO  [RS:1;0b0b0ae8de03:37509 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-17T20:35:52,748 INFO  [RS:1;0b0b0ae8de03:37509 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-17T20:35:52,748 INFO  [RS:1;0b0b0ae8de03:37509 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-17T20:35:52,748 INFO  [RS:1;0b0b0ae8de03:37509 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37509
2024-11-17T20:35:52,753 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1812) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$isFileClosed$57(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.isFileClosed(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1810) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:35:52,803 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:35:52,827 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:35:52,827 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:35:52,828 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:35:52,828 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:35:52,828 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:35:52,838 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:35:52,838 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:35:52,907 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37509-0x1014a8214ff0002, quorum=127.0.0.1:54994, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/0b0b0ae8de03,37509,1731875709020
2024-11-17T20:35:52,907 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40559-0x1014a8214ff0000, quorum=127.0.0.1:54994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-17T20:35:52,907 INFO  [RS:1;0b0b0ae8de03:37509 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-17T20:35:52,917 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [0b0b0ae8de03,37509,1731875709020]
2024-11-17T20:35:52,927 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/0b0b0ae8de03,37509,1731875709020 already deleted, retry=false
2024-11-17T20:35:52,927 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 0b0b0ae8de03,37509,1731875709020 expired; onlineServers=1
2024-11-17T20:35:52,935 ERROR [WAL-Shutdown-0 {}] wal.AbstractFSWAL(2118): We have waited 5 seconds but the close of async writer doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.fshlog.wait.on.shutdown.seconds"
2024-11-17T20:35:52,938 DEBUG [RS:0;0b0b0ae8de03:33133 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/oldWALs
2024-11-17T20:35:52,938 INFO  [RS:0;0b0b0ae8de03:33133 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 0b0b0ae8de03%2C33133%2C1731875707434.meta:.meta(num 1731875747734)
2024-11-17T20:35:52,938 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:35:52,938 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:35:52,938 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:35:52,938 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:35:52,938 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:35:52,940 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35007 is added to blk_1073741895_1079 (size=14682)
2024-11-17T20:35:52,941 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33949 is added to blk_1073741895_1079 (size=14682)
2024-11-17T20:35:52,943 DEBUG [RS:0;0b0b0ae8de03:33133 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/oldWALs
2024-11-17T20:35:52,943 INFO  [RS:0;0b0b0ae8de03:33133 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 0b0b0ae8de03%2C33133%2C1731875707434:(num 1731875747418)
2024-11-17T20:35:52,943 DEBUG [RS:0;0b0b0ae8de03:33133 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T20:35:52,943 INFO  [RS:0;0b0b0ae8de03:33133 {}] regionserver.LeaseManager(133): Closed leases
2024-11-17T20:35:52,943 INFO  [RS:0;0b0b0ae8de03:33133 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-17T20:35:52,944 INFO  [RS:0;0b0b0ae8de03:33133 {}] hbase.ChoreService(370): Chore service for: regionserver/0b0b0ae8de03:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown
2024-11-17T20:35:52,944 INFO  [RS:0;0b0b0ae8de03:33133 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-17T20:35:52,944 INFO  [regionserver/0b0b0ae8de03:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-17T20:35:52,944 INFO  [RS:0;0b0b0ae8de03:33133 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33133
2024-11-17T20:35:52,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40559-0x1014a8214ff0000, quorum=127.0.0.1:54994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-17T20:35:52,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33133-0x1014a8214ff0001, quorum=127.0.0.1:54994, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/0b0b0ae8de03,33133,1731875707434
2024-11-17T20:35:52,956 INFO  [RS:0;0b0b0ae8de03:33133 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-17T20:35:52,966 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [0b0b0ae8de03,33133,1731875707434]
2024-11-17T20:35:52,977 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/0b0b0ae8de03,33133,1731875707434 already deleted, retry=false
2024-11-17T20:35:52,977 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 0b0b0ae8de03,33133,1731875707434 expired; onlineServers=0
2024-11-17T20:35:52,977 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '0b0b0ae8de03,40559,1731875707247' *****
2024-11-17T20:35:52,977 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-17T20:35:52,977 INFO  [M:0;0b0b0ae8de03:40559 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-17T20:35:52,977 INFO  [M:0;0b0b0ae8de03:40559 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-17T20:35:52,977 DEBUG [M:0;0b0b0ae8de03:40559 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-17T20:35:52,977 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-17T20:35:52,977 DEBUG [M:0;0b0b0ae8de03:40559 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-17T20:35:52,977 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster-HFileCleaner.small.0-1731875707920 {}] cleaner.HFileCleaner(306): Exit Thread[master/0b0b0ae8de03:0:becomeActiveMaster-HFileCleaner.small.0-1731875707920,5,FailOnTimeoutGroup]
2024-11-17T20:35:52,977 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster-HFileCleaner.large.0-1731875707916 {}] cleaner.HFileCleaner(306): Exit Thread[master/0b0b0ae8de03:0:becomeActiveMaster-HFileCleaner.large.0-1731875707916,5,FailOnTimeoutGroup]
2024-11-17T20:35:52,978 INFO  [M:0;0b0b0ae8de03:40559 {}] hbase.ChoreService(370): Chore service for: master/0b0b0ae8de03:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-17T20:35:52,978 INFO  [M:0;0b0b0ae8de03:40559 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-17T20:35:52,978 DEBUG [M:0;0b0b0ae8de03:40559 {}] master.HMaster(1795): Stopping service threads
2024-11-17T20:35:52,978 INFO  [M:0;0b0b0ae8de03:40559 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-17T20:35:52,978 INFO  [M:0;0b0b0ae8de03:40559 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-17T20:35:52,978 INFO  [M:0;0b0b0ae8de03:40559 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-17T20:35:52,978 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-17T20:35:52,990 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40559-0x1014a8214ff0000, quorum=127.0.0.1:54994, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-17T20:35:52,990 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40559-0x1014a8214ff0000, quorum=127.0.0.1:54994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:35:52,991 DEBUG [M:0;0b0b0ae8de03:40559 {}] zookeeper.ZKUtil(347): master:40559-0x1014a8214ff0000, quorum=127.0.0.1:54994, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-17T20:35:52,991 WARN  [M:0;0b0b0ae8de03:40559 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-17T20:35:52,991 INFO  [M:0;0b0b0ae8de03:40559 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/.lastflushedseqids
2024-11-17T20:35:52,998 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35007 is added to blk_1073741916_1102 (size=130)
2024-11-17T20:35:52,998 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33949 is added to blk_1073741916_1102 (size=130)
2024-11-17T20:35:52,999 INFO  [M:0;0b0b0ae8de03:40559 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-17T20:35:52,999 INFO  [M:0;0b0b0ae8de03:40559 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-17T20:35:52,999 DEBUG [M:0;0b0b0ae8de03:40559 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-17T20:35:52,999 INFO  [M:0;0b0b0ae8de03:40559 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T20:35:52,999 DEBUG [M:0;0b0b0ae8de03:40559 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T20:35:52,999 DEBUG [M:0;0b0b0ae8de03:40559 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-17T20:35:52,999 DEBUG [M:0;0b0b0ae8de03:40559 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T20:35:53,000 INFO  [M:0;0b0b0ae8de03:40559 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.25 KB heapSize=29.49 KB
2024-11-17T20:35:53,017 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37509-0x1014a8214ff0002, quorum=127.0.0.1:54994, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-17T20:35:53,017 INFO  [RS:1;0b0b0ae8de03:37509 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-17T20:35:53,017 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37509-0x1014a8214ff0002, quorum=127.0.0.1:54994, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-17T20:35:53,017 INFO  [RS:1;0b0b0ae8de03:37509 {}] regionserver.HRegionServer(1031): Exiting; stopping=0b0b0ae8de03,37509,1731875709020; zookeeper connection closed.
2024-11-17T20:35:53,017 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7251c99 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7251c99
2024-11-17T20:35:53,021 DEBUG [M:0;0b0b0ae8de03:40559 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/cd318cae9f9843538439e4f53e9ed844 is 82, key is hbase:meta,,1/info:regioninfo/1731875708889/Put/seqid=0
2024-11-17T20:35:53,026 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33949 is added to blk_1073741917_1103 (size=5672)
2024-11-17T20:35:53,026 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35007 is added to blk_1073741917_1103 (size=5672)
2024-11-17T20:35:53,027 INFO  [M:0;0b0b0ae8de03:40559 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/cd318cae9f9843538439e4f53e9ed844
2024-11-17T20:35:53,052 DEBUG [M:0;0b0b0ae8de03:40559 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ad89620b9e4f40ba87c0cad6386799cf is 774, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731875710014/Put/seqid=0
2024-11-17T20:35:53,057 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33949 is added to blk_1073741918_1104 (size=6255)
2024-11-17T20:35:53,057 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35007 is added to blk_1073741918_1104 (size=6255)
2024-11-17T20:35:53,058 INFO  [M:0;0b0b0ae8de03:40559 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.58 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ad89620b9e4f40ba87c0cad6386799cf
2024-11-17T20:35:53,064 INFO  [M:0;0b0b0ae8de03:40559 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ad89620b9e4f40ba87c0cad6386799cf
2024-11-17T20:35:53,066 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33133-0x1014a8214ff0001, quorum=127.0.0.1:54994, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-17T20:35:53,066 INFO  [RS:0;0b0b0ae8de03:33133 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-17T20:35:53,067 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33133-0x1014a8214ff0001, quorum=127.0.0.1:54994, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-17T20:35:53,067 INFO  [RS:0;0b0b0ae8de03:33133 {}] regionserver.HRegionServer(1031): Exiting; stopping=0b0b0ae8de03,33133,1731875707434; zookeeper connection closed.
2024-11-17T20:35:53,067 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5ee4eff7 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5ee4eff7
2024-11-17T20:35:53,067 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete
2024-11-17T20:35:53,081 DEBUG [M:0;0b0b0ae8de03:40559 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5ab179b214e343469f50b53dd798a90e is 69, key is 0b0b0ae8de03,33133,1731875707434/rs:state/1731875708057/Put/seqid=0
2024-11-17T20:35:53,086 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33949 is added to blk_1073741919_1105 (size=5224)
2024-11-17T20:35:53,086 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35007 is added to blk_1073741919_1105 (size=5224)
2024-11-17T20:35:53,087 INFO  [M:0;0b0b0ae8de03:40559 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5ab179b214e343469f50b53dd798a90e
2024-11-17T20:35:53,109 DEBUG [M:0;0b0b0ae8de03:40559 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/8d8aaab17ee0498aa1878fd81e34cc4a is 52, key is load_balancer_on/state:d/1731875708996/Put/seqid=0
2024-11-17T20:35:53,114 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35007 is added to blk_1073741920_1106 (size=5056)
2024-11-17T20:35:53,114 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33949 is added to blk_1073741920_1106 (size=5056)
2024-11-17T20:35:53,115 INFO  [M:0;0b0b0ae8de03:40559 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/8d8aaab17ee0498aa1878fd81e34cc4a
2024-11-17T20:35:53,122 DEBUG [M:0;0b0b0ae8de03:40559 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/cd318cae9f9843538439e4f53e9ed844 as hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/cd318cae9f9843538439e4f53e9ed844
2024-11-17T20:35:53,128 INFO  [M:0;0b0b0ae8de03:40559 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/cd318cae9f9843538439e4f53e9ed844, entries=8, sequenceid=60, filesize=5.5 K
2024-11-17T20:35:53,129 DEBUG [M:0;0b0b0ae8de03:40559 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ad89620b9e4f40ba87c0cad6386799cf as hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/ad89620b9e4f40ba87c0cad6386799cf
2024-11-17T20:35:53,136 INFO  [M:0;0b0b0ae8de03:40559 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ad89620b9e4f40ba87c0cad6386799cf
2024-11-17T20:35:53,136 INFO  [M:0;0b0b0ae8de03:40559 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/ad89620b9e4f40ba87c0cad6386799cf, entries=6, sequenceid=60, filesize=6.1 K
2024-11-17T20:35:53,138 DEBUG [M:0;0b0b0ae8de03:40559 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5ab179b214e343469f50b53dd798a90e as hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/5ab179b214e343469f50b53dd798a90e
2024-11-17T20:35:53,144 INFO  [M:0;0b0b0ae8de03:40559 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/5ab179b214e343469f50b53dd798a90e, entries=2, sequenceid=60, filesize=5.1 K
2024-11-17T20:35:53,146 DEBUG [M:0;0b0b0ae8de03:40559 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/8d8aaab17ee0498aa1878fd81e34cc4a as hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/8d8aaab17ee0498aa1878fd81e34cc4a
2024-11-17T20:35:53,151 INFO  [M:0;0b0b0ae8de03:40559 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/8d8aaab17ee0498aa1878fd81e34cc4a, entries=1, sequenceid=60, filesize=4.9 K
2024-11-17T20:35:53,152 INFO  [M:0;0b0b0ae8de03:40559 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.25 KB/23805, heapSize ~29.43 KB/30136, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 153ms, sequenceid=60, compaction requested=false
2024-11-17T20:35:53,154 INFO  [M:0;0b0b0ae8de03:40559 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T20:35:53,154 DEBUG [M:0;0b0b0ae8de03:40559 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731875752999Disabling compacts and flushes for region at 1731875752999Disabling writes for close at 1731875752999Obtaining lock to block concurrent updates at 1731875753000 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731875753000Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23805, getHeapSize=30136, getOffHeapSize=0, getCellsCount=71 at 1731875753000Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731875753001 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731875753001Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731875753020 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731875753020Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731875753032 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731875753051 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731875753051Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731875753064 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731875753081 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731875753081Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731875753093 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731875753109 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731875753109Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@214c326: reopening flushed file at 1731875753121 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@472798fa: reopening flushed file at 1731875753128 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@722b97ca: reopening flushed file at 1731875753137 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@18c696b6: reopening flushed file at 1731875753145 (+8 ms)Finished flush of dataSize ~23.25 KB/23805, heapSize ~29.43 KB/30136, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 153ms, sequenceid=60, compaction requested=false at 1731875753153 (+8 ms)Writing region close event to WAL at 1731875753154 (+1 ms)Closed at 1731875753154
2024-11-17T20:35:53,154 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:35:53,154 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:35:53,154 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:35:53,155 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:35:53,155 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:35:53,157 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35007 is added to blk_1073741891_1074 (size=1045)
2024-11-17T20:35:53,157 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33949 is added to blk_1073741891_1074 (size=1045)
2024-11-17T20:35:53,341 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-17T20:35:53,379 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:35:53,379 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:35:53,380 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:35:53,380 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:35:53,380 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:35:53,381 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:35:53,387 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:35:53,390 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:35:53,754 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:35:53,772 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:35:53,803 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath
2024-11-17T20:35:53,803 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-17T20:35:53,804 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-17T20:35:53,804 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-11-17T20:35:54,371 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35007 is added to blk_1073741833_1009 (size=32)
2024-11-17T20:35:54,371 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35007 is added to blk_1073741831_1007 (size=1321)
2024-11-17T20:35:54,755 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:35:54,773 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:35:55,370 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35007 is added to blk_1073741829_1005 (size=34)
2024-11-17T20:35:55,370 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35007 is added to blk_1073741827_1003 (size=196)
2024-11-17T20:35:55,756 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:35:55,773 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:35:56,757 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:35:56,774 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:35:57,372 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35007 is added to blk_1073741825_1001 (size=7)
2024-11-17T20:35:57,758 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:35:57,775 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:35:58,155 ERROR [WAL-Shutdown-0 {}] wal.AbstractFSWAL(2118): We have waited 5 seconds but the close of async writer doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.fshlog.wait.on.shutdown.seconds"
2024-11-17T20:35:58,159 INFO  [M:0;0b0b0ae8de03:40559 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-17T20:35:58,159 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-17T20:35:58,160 INFO  [M:0;0b0b0ae8de03:40559 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40559
2024-11-17T20:35:58,160 INFO  [M:0;0b0b0ae8de03:40559 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-17T20:35:58,311 INFO  [M:0;0b0b0ae8de03:40559 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-17T20:35:58,312 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40559-0x1014a8214ff0000, quorum=127.0.0.1:54994, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-17T20:35:58,312 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40559-0x1014a8214ff0000, quorum=127.0.0.1:54994, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-17T20:35:58,353 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3a8cd6ef{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T20:35:58,353 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@25f6ac6c{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-17T20:35:58,353 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-17T20:35:58,353 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@117da513{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-17T20:35:58,354 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1b1c2893{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/hadoop.log.dir/,STOPPED}
2024-11-17T20:35:58,355 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-17T20:35:58,355 WARN  [BP-1477976244-172.17.0.2-1731875704920 heartbeating to localhost/127.0.0.1:33551 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-17T20:35:58,355 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-17T20:35:58,355 WARN  [BP-1477976244-172.17.0.2-1731875704920 heartbeating to localhost/127.0.0.1:33551 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1477976244-172.17.0.2-1731875704920 (Datanode Uuid 6506fff5-03d5-479e-8b59-d4ac07be411d) service to localhost/127.0.0.1:33551
2024-11-17T20:35:58,355 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@58a6b258 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1477976244-172.17.0.2-1731875704920:blk_1073741836_1012, datanode=DatanodeInfoWithStorage[127.0.0.1:41145,null,null])
java.io.InterruptedIOException: DestHost:destPort localhost:42067 , LocalHost:localPort 0b0b0ae8de03/172.17.0.2:0. Failed on local exception: java.io.InterruptedIOException: Interrupted: action=RetryAction(action=RETRY, delayMillis=1000, reason=retries get failed due to exceeded maximum allowed retries number: 10), retry policy=RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS)
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:936) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.InterruptedIOException: Interrupted: action=RetryAction(action=RETRY, delayMillis=1000, reason=retries get failed due to exceeded maximum allowed retries number: 10), retry policy=RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS)
	at org.apache.hadoop.ipc.Client$Connection.handleConnectionFailure(Client.java:963) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:691) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
Caused by: java.lang.InterruptedException: sleep interrupted
	at java.lang.Thread.sleep(Native Method) ~[?:?]
	at org.apache.hadoop.ipc.Client$Connection.handleConnectionFailure(Client.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:691) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
2024-11-17T20:35:58,355 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@58a6b258 {}] datanode.BlockRecoveryWorker$1(605): recover Block: RecoveringBlock{BP-1477976244-172.17.0.2-1731875704920:blk_1073741836_1012; getBlockSize()=85; corrupt=false; offset=-1; locs=[DatanodeInfoWithStorage[127.0.0.1:33949,null,null], DatanodeInfoWithStorage[127.0.0.1:41145,null,null]]; cachedLocs=[]} FAILED: java.io.IOException: No block pool offer service for bpid=BP-1477976244-172.17.0.2-1731875704920
2024-11-17T20:35:58,356 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@58a6b258 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1477976244-172.17.0.2-1731875704920:blk_1073741834_1010, datanode=DatanodeInfoWithStorage[127.0.0.1:41145,null,null])
java.io.IOException: No block pool offer service for bpid=BP-1477976244-172.17.0.2-1731875704920
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.getDatanodeID(BlockRecoveryWorker.java:539) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$000(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:131) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:35:58,356 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/cluster_b9ca40e8-4224-d8ab-7466-d1512b745e25/data/data3/current/BP-1477976244-172.17.0.2-1731875704920 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T20:35:58,356 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@58a6b258 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1477976244-172.17.0.2-1731875704920:blk_1073741834_1010, datanode=DatanodeInfoWithStorage[127.0.0.1:33949,null,null])
java.io.IOException: No block pool offer service for bpid=BP-1477976244-172.17.0.2-1731875704920
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.getDatanodeID(BlockRecoveryWorker.java:539) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$000(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:131) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:35:58,356 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@58a6b258 {}] datanode.BlockRecoveryWorker$1(605): recover Block: RecoveringBlock{BP-1477976244-172.17.0.2-1731875704920:blk_1073741834_1010; getBlockSize()=85; corrupt=false; offset=-1; locs=[DatanodeInfoWithStorage[127.0.0.1:41145,null,null], DatanodeInfoWithStorage[127.0.0.1:33949,null,null]]; cachedLocs=[]} FAILED: java.io.IOException: All datanodes failed: block=BP-1477976244-172.17.0.2-1731875704920:blk_1073741834_1010, datanodeids=[DatanodeInfoWithStorage[127.0.0.1:41145,null,null], DatanodeInfoWithStorage[127.0.0.1:33949,null,null]]
2024-11-17T20:35:58,356 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/cluster_b9ca40e8-4224-d8ab-7466-d1512b745e25/data/data4/current/BP-1477976244-172.17.0.2-1731875704920 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T20:35:58,356 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-17T20:35:58,359 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@442cf16f{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T20:35:58,359 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5ec5aa68{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-17T20:35:58,359 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-17T20:35:58,360 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7d285dba{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-17T20:35:58,360 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@92842f7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/hadoop.log.dir/,STOPPED}
2024-11-17T20:35:58,361 WARN  [BP-1477976244-172.17.0.2-1731875704920 heartbeating to localhost/127.0.0.1:33551 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-17T20:35:58,361 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-17T20:35:58,361 WARN  [BP-1477976244-172.17.0.2-1731875704920 heartbeating to localhost/127.0.0.1:33551 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1477976244-172.17.0.2-1731875704920 (Datanode Uuid 96f27179-3395-467d-8cec-ee4e6e5e306e) service to localhost/127.0.0.1:33551
2024-11-17T20:35:58,361 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-17T20:35:58,362 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/cluster_b9ca40e8-4224-d8ab-7466-d1512b745e25/data/data9/current/BP-1477976244-172.17.0.2-1731875704920 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T20:35:58,362 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/cluster_b9ca40e8-4224-d8ab-7466-d1512b745e25/data/data10/current/BP-1477976244-172.17.0.2-1731875704920 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T20:35:58,362 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-17T20:35:58,368 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@66182b08{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-17T20:35:58,368 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1100af17{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-17T20:35:58,368 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-17T20:35:58,368 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@8f6e525{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-17T20:35:58,368 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1d790455{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/hadoop.log.dir/,STOPPED}
2024-11-17T20:35:58,376 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-17T20:35:58,414 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-17T20:35:58,425 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnDatanodeDeath Thread=158 (was 82)
Potentially hanging thread: HMaster-EventLoopGroup-7-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-5-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-18-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-18-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-7-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-20-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.2@localhost:33551
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-21-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.1@localhost:36611
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:33551
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Close-WAL-Writer-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:175)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$899/0x00007fba44bf4000.run(Unknown Source)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-19-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:33551
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:33551
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Timer for 'DataNode' metrics system
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-20-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-20-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-21-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:33551 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-19-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-7-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-19-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-21-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:33551
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:33551
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-18-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:33551 from jenkins.hfs.3
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: Close-WAL-Writer-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:175)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$899/0x00007fba44bf4000.run(Unknown Source)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-5-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:36611
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:33551 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:33551 from jenkins.hfs.2
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.3@localhost:33551
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-5-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Close-WAL-Writer-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:175)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$899/0x00007fba44bf4000.run(Unknown Source)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=438 (was 404) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=221 (was 176) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7767 (was 8632)
2024-11-17T20:35:58,433 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnPipelineRestart Thread=158, OpenFileDescriptor=438, MaxFileDescriptor=1048576, SystemLoadAverage=221, ProcessCount=11, AvailableMemoryMB=7767
2024-11-17T20:35:58,433 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-17T20:35:58,434 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/hadoop.log.dir so I do NOT create it in target/test-data/d2981510-59e6-ff69-efd1-5906a8a92785
2024-11-17T20:35:58,434 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5283882e-a771-ecd7-cb05-45ef84522c25/hadoop.tmp.dir so I do NOT create it in target/test-data/d2981510-59e6-ff69-efd1-5906a8a92785
2024-11-17T20:35:58,434 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d2981510-59e6-ff69-efd1-5906a8a92785/cluster_a0b2892d-6b1d-843d-7c12-ddaf29516f86, deleteOnExit=true
2024-11-17T20:35:58,434 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-17T20:35:58,434 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d2981510-59e6-ff69-efd1-5906a8a92785/test.cache.data in system properties and HBase conf
2024-11-17T20:35:58,434 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d2981510-59e6-ff69-efd1-5906a8a92785/hadoop.tmp.dir in system properties and HBase conf
2024-11-17T20:35:58,434 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d2981510-59e6-ff69-efd1-5906a8a92785/hadoop.log.dir in system properties and HBase conf
2024-11-17T20:35:58,434 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d2981510-59e6-ff69-efd1-5906a8a92785/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-17T20:35:58,434 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d2981510-59e6-ff69-efd1-5906a8a92785/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-17T20:35:58,434 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-17T20:35:58,434 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-17T20:35:58,435 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d2981510-59e6-ff69-efd1-5906a8a92785/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-17T20:35:58,435 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d2981510-59e6-ff69-efd1-5906a8a92785/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-17T20:35:58,435 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d2981510-59e6-ff69-efd1-5906a8a92785/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-17T20:35:58,435 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d2981510-59e6-ff69-efd1-5906a8a92785/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-17T20:35:58,435 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d2981510-59e6-ff69-efd1-5906a8a92785/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-17T20:35:58,435 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d2981510-59e6-ff69-efd1-5906a8a92785/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-17T20:35:58,435 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d2981510-59e6-ff69-efd1-5906a8a92785/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-17T20:35:58,435 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d2981510-59e6-ff69-efd1-5906a8a92785/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-17T20:35:58,435 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d2981510-59e6-ff69-efd1-5906a8a92785/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-17T20:35:58,435 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d2981510-59e6-ff69-efd1-5906a8a92785/nfs.dump.dir in system properties and HBase conf
2024-11-17T20:35:58,436 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d2981510-59e6-ff69-efd1-5906a8a92785/java.io.tmpdir in system properties and HBase conf
2024-11-17T20:35:58,436 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d2981510-59e6-ff69-efd1-5906a8a92785/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-17T20:35:58,436 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d2981510-59e6-ff69-efd1-5906a8a92785/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-17T20:35:58,436 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d2981510-59e6-ff69-efd1-5906a8a92785/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-17T20:35:58,449 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-17T20:35:58,759 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:35:58,776 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:35:58,846 WARN  [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
2024-11-17T20:35:58,863 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:35:58,863 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:35:58,864 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:35:58,864 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:35:58,864 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:35:58,865 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:35:58,868 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:35:58,869 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:35:58,869 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:35:58,871 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:35:58,890 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-17T20:35:58,895 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-17T20:35:58,896 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-17T20:35:58,897 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-17T20:35:58,897 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-17T20:35:58,897 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-17T20:35:58,898 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@179d1ca6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d2981510-59e6-ff69-efd1-5906a8a92785/hadoop.log.dir/,AVAILABLE}
2024-11-17T20:35:58,898 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2a99c341{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-17T20:35:58,932 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:35:59,022 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7c6cb23c{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d2981510-59e6-ff69-efd1-5906a8a92785/java.io.tmpdir/jetty-localhost-39811-hadoop-hdfs-3_4_1-tests_jar-_-any-4054833133717868274/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-17T20:35:59,022 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@38a385f1{HTTP/1.1, (http/1.1)}{localhost:39811}
2024-11-17T20:35:59,022 INFO  [Time-limited test {}] server.Server(415): Started @162308ms
2024-11-17T20:35:59,035 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-17T20:35:59,304 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-17T20:35:59,310 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-17T20:35:59,312 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-17T20:35:59,313 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-17T20:35:59,313 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-17T20:35:59,315 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@667c8bc{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d2981510-59e6-ff69-efd1-5906a8a92785/hadoop.log.dir/,AVAILABLE}
2024-11-17T20:35:59,316 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@31e2f2e1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-17T20:35:59,455 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@68f9eaf6{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d2981510-59e6-ff69-efd1-5906a8a92785/java.io.tmpdir/jetty-localhost-36039-hadoop-hdfs-3_4_1-tests_jar-_-any-4123763088818843731/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T20:35:59,455 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6b8d1f8b{HTTP/1.1, (http/1.1)}{localhost:36039}
2024-11-17T20:35:59,456 INFO  [Time-limited test {}] server.Server(415): Started @162742ms
2024-11-17T20:35:59,458 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-17T20:35:59,516 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-17T20:35:59,521 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-17T20:35:59,525 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-17T20:35:59,525 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-17T20:35:59,525 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-17T20:35:59,526 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@40375d59{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d2981510-59e6-ff69-efd1-5906a8a92785/hadoop.log.dir/,AVAILABLE}
2024-11-17T20:35:59,527 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@68c19d09{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-17T20:35:59,639 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@32ef6c57{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d2981510-59e6-ff69-efd1-5906a8a92785/java.io.tmpdir/jetty-localhost-41471-hadoop-hdfs-3_4_1-tests_jar-_-any-4023962504584086693/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T20:35:59,639 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3c9f0fbb{HTTP/1.1, (http/1.1)}{localhost:41471}
2024-11-17T20:35:59,639 INFO  [Time-limited test {}] server.Server(415): Started @162925ms
2024-11-17T20:35:59,640 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-17T20:35:59,759 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:35:59,777 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:35:59,933 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:00,760 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:00,777 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:00,814 WARN  [Thread-1209 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d2981510-59e6-ff69-efd1-5906a8a92785/cluster_a0b2892d-6b1d-843d-7c12-ddaf29516f86/data/data2/current/BP-1977883270-172.17.0.2-1731875758461/current, will proceed with Du for space computation calculation, 
2024-11-17T20:36:00,814 WARN  [Thread-1208 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d2981510-59e6-ff69-efd1-5906a8a92785/cluster_a0b2892d-6b1d-843d-7c12-ddaf29516f86/data/data1/current/BP-1977883270-172.17.0.2-1731875758461/current, will proceed with Du for space computation calculation, 
2024-11-17T20:36:00,836 WARN  [Thread-1172 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-17T20:36:00,843 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xbd08bd99193ae738 with lease ID 0xfece477006bdaaca: Processing first storage report for DS-5091058c-e4c0-4649-ad99-02c88df21742 from datanode DatanodeRegistration(127.0.0.1:45599, datanodeUuid=ba381e1e-86b5-491b-b28d-cf1370bb3873, infoPort=34383, infoSecurePort=0, ipcPort=40025, storageInfo=lv=-57;cid=testClusterID;nsid=566398173;c=1731875758461)
2024-11-17T20:36:00,843 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbd08bd99193ae738 with lease ID 0xfece477006bdaaca: from storage DS-5091058c-e4c0-4649-ad99-02c88df21742 node DatanodeRegistration(127.0.0.1:45599, datanodeUuid=ba381e1e-86b5-491b-b28d-cf1370bb3873, infoPort=34383, infoSecurePort=0, ipcPort=40025, storageInfo=lv=-57;cid=testClusterID;nsid=566398173;c=1731875758461), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-17T20:36:00,843 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xbd08bd99193ae738 with lease ID 0xfece477006bdaaca: Processing first storage report for DS-06de2f39-3030-45da-be94-2451d740ea4b from datanode DatanodeRegistration(127.0.0.1:45599, datanodeUuid=ba381e1e-86b5-491b-b28d-cf1370bb3873, infoPort=34383, infoSecurePort=0, ipcPort=40025, storageInfo=lv=-57;cid=testClusterID;nsid=566398173;c=1731875758461)
2024-11-17T20:36:00,843 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbd08bd99193ae738 with lease ID 0xfece477006bdaaca: from storage DS-06de2f39-3030-45da-be94-2451d740ea4b node DatanodeRegistration(127.0.0.1:45599, datanodeUuid=ba381e1e-86b5-491b-b28d-cf1370bb3873, infoPort=34383, infoSecurePort=0, ipcPort=40025, storageInfo=lv=-57;cid=testClusterID;nsid=566398173;c=1731875758461), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-17T20:36:00,933 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:00,953 WARN  [Thread-1219 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d2981510-59e6-ff69-efd1-5906a8a92785/cluster_a0b2892d-6b1d-843d-7c12-ddaf29516f86/data/data3/current/BP-1977883270-172.17.0.2-1731875758461/current, will proceed with Du for space computation calculation, 
2024-11-17T20:36:00,954 WARN  [Thread-1220 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d2981510-59e6-ff69-efd1-5906a8a92785/cluster_a0b2892d-6b1d-843d-7c12-ddaf29516f86/data/data4/current/BP-1977883270-172.17.0.2-1731875758461/current, will proceed with Du for space computation calculation, 
2024-11-17T20:36:00,986 WARN  [Thread-1195 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-17T20:36:00,991 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x18a4fc05e77313e0 with lease ID 0xfece477006bdaacb: Processing first storage report for DS-658f96ea-3d36-4123-8e44-dc53dc482242 from datanode DatanodeRegistration(127.0.0.1:46273, datanodeUuid=35b41824-7467-4b8c-81a8-a7878b27f81a, infoPort=38015, infoSecurePort=0, ipcPort=41333, storageInfo=lv=-57;cid=testClusterID;nsid=566398173;c=1731875758461)
2024-11-17T20:36:00,991 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x18a4fc05e77313e0 with lease ID 0xfece477006bdaacb: from storage DS-658f96ea-3d36-4123-8e44-dc53dc482242 node DatanodeRegistration(127.0.0.1:46273, datanodeUuid=35b41824-7467-4b8c-81a8-a7878b27f81a, infoPort=38015, infoSecurePort=0, ipcPort=41333, storageInfo=lv=-57;cid=testClusterID;nsid=566398173;c=1731875758461), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-17T20:36:00,992 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x18a4fc05e77313e0 with lease ID 0xfece477006bdaacb: Processing first storage report for DS-314bf586-6b8c-4cfe-8550-7b7d9a017aef from datanode DatanodeRegistration(127.0.0.1:46273, datanodeUuid=35b41824-7467-4b8c-81a8-a7878b27f81a, infoPort=38015, infoSecurePort=0, ipcPort=41333, storageInfo=lv=-57;cid=testClusterID;nsid=566398173;c=1731875758461)
2024-11-17T20:36:00,992 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x18a4fc05e77313e0 with lease ID 0xfece477006bdaacb: from storage DS-314bf586-6b8c-4cfe-8550-7b7d9a017aef node DatanodeRegistration(127.0.0.1:46273, datanodeUuid=35b41824-7467-4b8c-81a8-a7878b27f81a, infoPort=38015, infoSecurePort=0, ipcPort=41333, storageInfo=lv=-57;cid=testClusterID;nsid=566398173;c=1731875758461), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-17T20:36:01,093 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d2981510-59e6-ff69-efd1-5906a8a92785
2024-11-17T20:36:01,096 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d2981510-59e6-ff69-efd1-5906a8a92785/cluster_a0b2892d-6b1d-843d-7c12-ddaf29516f86/zookeeper_0, clientPort=49481, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d2981510-59e6-ff69-efd1-5906a8a92785/cluster_a0b2892d-6b1d-843d-7c12-ddaf29516f86/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d2981510-59e6-ff69-efd1-5906a8a92785/cluster_a0b2892d-6b1d-843d-7c12-ddaf29516f86/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-17T20:36:01,097 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=49481
2024-11-17T20:36:01,098 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T20:36:01,099 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T20:36:01,118 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741825_1001 (size=7)
2024-11-17T20:36:01,119 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46273 is added to blk_1073741825_1001 (size=7)
2024-11-17T20:36:01,120 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6 with version=8
2024-11-17T20:36:01,120 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/hbase-staging
2024-11-17T20:36:01,122 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/0b0b0ae8de03:0 server-side Connection retries=45
2024-11-17T20:36:01,122 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-17T20:36:01,122 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-17T20:36:01,122 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-17T20:36:01,122 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-17T20:36:01,122 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-17T20:36:01,122 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-17T20:36:01,122 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-17T20:36:01,123 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36429
2024-11-17T20:36:01,124 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:36429 connecting to ZooKeeper ensemble=127.0.0.1:49481
2024-11-17T20:36:01,184 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:364290x0, quorum=127.0.0.1:49481, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-17T20:36:01,184 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36429-0x1014a82e7730000 connected
2024-11-17T20:36:01,270 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T20:36:01,272 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T20:36:01,274 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36429-0x1014a82e7730000, quorum=127.0.0.1:49481, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-17T20:36:01,274 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6, hbase.cluster.distributed=false
2024-11-17T20:36:01,276 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36429-0x1014a82e7730000, quorum=127.0.0.1:49481, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-17T20:36:01,276 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36429
2024-11-17T20:36:01,276 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36429
2024-11-17T20:36:01,277 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36429
2024-11-17T20:36:01,282 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36429
2024-11-17T20:36:01,282 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36429
2024-11-17T20:36:01,298 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/0b0b0ae8de03:0 server-side Connection retries=45
2024-11-17T20:36:01,298 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-17T20:36:01,298 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-17T20:36:01,299 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-17T20:36:01,299 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-17T20:36:01,299 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-17T20:36:01,299 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-17T20:36:01,299 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-17T20:36:01,299 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35963
2024-11-17T20:36:01,301 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35963 connecting to ZooKeeper ensemble=127.0.0.1:49481
2024-11-17T20:36:01,302 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T20:36:01,304 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T20:36:01,312 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:359630x0, quorum=127.0.0.1:49481, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-17T20:36:01,313 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35963-0x1014a82e7730001 connected
2024-11-17T20:36:01,313 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35963-0x1014a82e7730001, quorum=127.0.0.1:49481, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-17T20:36:01,313 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-17T20:36:01,317 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-17T20:36:01,318 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35963-0x1014a82e7730001, quorum=127.0.0.1:49481, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-17T20:36:01,319 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35963-0x1014a82e7730001, quorum=127.0.0.1:49481, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-17T20:36:01,320 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35963
2024-11-17T20:36:01,321 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35963
2024-11-17T20:36:01,324 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35963
2024-11-17T20:36:01,325 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35963
2024-11-17T20:36:01,325 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35963
2024-11-17T20:36:01,337 DEBUG [M:0;0b0b0ae8de03:36429 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;0b0b0ae8de03:36429
2024-11-17T20:36:01,338 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/0b0b0ae8de03,36429,1731875761121
2024-11-17T20:36:01,344 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36429-0x1014a82e7730000, quorum=127.0.0.1:49481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-17T20:36:01,344 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35963-0x1014a82e7730001, quorum=127.0.0.1:49481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-17T20:36:01,344 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36429-0x1014a82e7730000, quorum=127.0.0.1:49481, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/0b0b0ae8de03,36429,1731875761121
2024-11-17T20:36:01,354 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35963-0x1014a82e7730001, quorum=127.0.0.1:49481, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-17T20:36:01,354 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36429-0x1014a82e7730000, quorum=127.0.0.1:49481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:36:01,354 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35963-0x1014a82e7730001, quorum=127.0.0.1:49481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:36:01,355 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36429-0x1014a82e7730000, quorum=127.0.0.1:49481, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-17T20:36:01,355 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/0b0b0ae8de03,36429,1731875761121 from backup master directory
2024-11-17T20:36:01,365 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35963-0x1014a82e7730001, quorum=127.0.0.1:49481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-17T20:36:01,365 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36429-0x1014a82e7730000, quorum=127.0.0.1:49481, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/0b0b0ae8de03,36429,1731875761121
2024-11-17T20:36:01,365 WARN  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-17T20:36:01,365 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36429-0x1014a82e7730000, quorum=127.0.0.1:49481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-17T20:36:01,365 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=0b0b0ae8de03,36429,1731875761121
2024-11-17T20:36:01,370 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/hbase.id] with ID: a1b4dec0-8444-4d02-83de-c771e6b96f93
2024-11-17T20:36:01,370 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/.tmp/hbase.id
2024-11-17T20:36:01,376 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46273 is added to blk_1073741826_1002 (size=42)
2024-11-17T20:36:01,376 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741826_1002 (size=42)
2024-11-17T20:36:01,377 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/.tmp/hbase.id]:[hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/hbase.id]
2024-11-17T20:36:01,395 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T20:36:01,395 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-17T20:36:01,398 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 3ms.
2024-11-17T20:36:01,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35963-0x1014a82e7730001, quorum=127.0.0.1:49481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:36:01,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36429-0x1014a82e7730000, quorum=127.0.0.1:49481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:36:01,425 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741827_1003 (size=196)
2024-11-17T20:36:01,426 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46273 is added to blk_1073741827_1003 (size=196)
2024-11-17T20:36:01,761 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:01,778 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:01,827 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-17T20:36:01,828 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-17T20:36:01,828 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-17T20:36:01,842 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46273 is added to blk_1073741828_1004 (size=1189)
2024-11-17T20:36:01,842 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741828_1004 (size=1189)
2024-11-17T20:36:01,843 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/MasterData/data/master/store
2024-11-17T20:36:01,850 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46273 is added to blk_1073741829_1005 (size=34)
2024-11-17T20:36:01,850 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741829_1005 (size=34)
2024-11-17T20:36:01,851 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T20:36:01,851 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-17T20:36:01,851 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T20:36:01,851 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T20:36:01,851 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-17T20:36:01,851 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T20:36:01,851 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T20:36:01,851 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731875761851Disabling compacts and flushes for region at 1731875761851Disabling writes for close at 1731875761851Writing region close event to WAL at 1731875761851Closed at 1731875761851
2024-11-17T20:36:01,852 WARN  [master/0b0b0ae8de03:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/MasterData/data/master/store/.initializing
2024-11-17T20:36:01,852 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/MasterData/WALs/0b0b0ae8de03,36429,1731875761121
2024-11-17T20:36:01,855 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0b0b0ae8de03%2C36429%2C1731875761121, suffix=, logDir=hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/MasterData/WALs/0b0b0ae8de03,36429,1731875761121, archiveDir=hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/MasterData/oldWALs, maxLogs=10
2024-11-17T20:36:01,855 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0b0b0ae8de03%2C36429%2C1731875761121.1731875761855
2024-11-17T20:36:01,860 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/MasterData/WALs/0b0b0ae8de03,36429,1731875761121/0b0b0ae8de03%2C36429%2C1731875761121.1731875761855
2024-11-17T20:36:01,865 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38015:38015),(127.0.0.1/127.0.0.1:34383:34383)]
2024-11-17T20:36:01,866 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-17T20:36:01,867 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T20:36:01,867 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T20:36:01,867 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T20:36:01,869 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-17T20:36:01,870 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-17T20:36:01,870 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:36:01,870 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T20:36:01,871 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-17T20:36:01,872 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-17T20:36:01,872 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:36:01,872 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-17T20:36:01,872 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-17T20:36:01,873 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-17T20:36:01,874 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:36:01,874 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-17T20:36:01,874 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-17T20:36:01,876 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-17T20:36:01,876 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:36:01,877 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-17T20:36:01,877 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T20:36:01,878 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-17T20:36:01,878 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-17T20:36:01,880 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T20:36:01,880 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T20:36:01,881 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-17T20:36:01,882 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T20:36:01,884 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-17T20:36:01,885 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=744333, jitterRate=-0.053532153367996216}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-17T20:36:01,886 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1731875761867Initializing all the Stores at 1731875761868 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731875761868Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731875761869 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731875761869Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731875761869Cleaning up temporary data from old regions at 1731875761880 (+11 ms)Region opened successfully at 1731875761886 (+6 ms)
2024-11-17T20:36:01,886 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-17T20:36:01,890 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2702e748, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0b0b0ae8de03/172.17.0.2:0
2024-11-17T20:36:01,891 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-17T20:36:01,891 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-17T20:36:01,891 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-17T20:36:01,891 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-17T20:36:01,892 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-17T20:36:01,892 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-17T20:36:01,892 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-17T20:36:01,895 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-17T20:36:01,896 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36429-0x1014a82e7730000, quorum=127.0.0.1:49481, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-17T20:36:01,934 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:01,942 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-17T20:36:01,942 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-17T20:36:01,943 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36429-0x1014a82e7730000, quorum=127.0.0.1:49481, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-17T20:36:01,954 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-17T20:36:01,955 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-17T20:36:01,955 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36429-0x1014a82e7730000, quorum=127.0.0.1:49481, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-17T20:36:01,965 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-17T20:36:01,966 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36429-0x1014a82e7730000, quorum=127.0.0.1:49481, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-17T20:36:01,975 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-17T20:36:01,978 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36429-0x1014a82e7730000, quorum=127.0.0.1:49481, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-17T20:36:01,986 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-17T20:36:01,996 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35963-0x1014a82e7730001, quorum=127.0.0.1:49481, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-17T20:36:01,996 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36429-0x1014a82e7730000, quorum=127.0.0.1:49481, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-17T20:36:01,996 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35963-0x1014a82e7730001, quorum=127.0.0.1:49481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:36:01,997 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36429-0x1014a82e7730000, quorum=127.0.0.1:49481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:36:01,997 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=0b0b0ae8de03,36429,1731875761121, sessionid=0x1014a82e7730000, setting cluster-up flag (Was=false)
2024-11-17T20:36:02,017 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36429-0x1014a82e7730000, quorum=127.0.0.1:49481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:36:02,017 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35963-0x1014a82e7730001, quorum=127.0.0.1:49481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:36:02,049 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-17T20:36:02,050 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0b0b0ae8de03,36429,1731875761121
2024-11-17T20:36:02,070 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35963-0x1014a82e7730001, quorum=127.0.0.1:49481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:36:02,070 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36429-0x1014a82e7730000, quorum=127.0.0.1:49481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:36:02,102 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-17T20:36:02,103 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0b0b0ae8de03,36429,1731875761121
2024-11-17T20:36:02,104 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-17T20:36:02,106 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-17T20:36:02,106 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-17T20:36:02,106 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-17T20:36:02,107 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 0b0b0ae8de03,36429,1731875761121
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-17T20:36:02,108 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/0b0b0ae8de03:0, corePoolSize=5, maxPoolSize=5
2024-11-17T20:36:02,108 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/0b0b0ae8de03:0, corePoolSize=5, maxPoolSize=5
2024-11-17T20:36:02,108 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/0b0b0ae8de03:0, corePoolSize=5, maxPoolSize=5
2024-11-17T20:36:02,108 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/0b0b0ae8de03:0, corePoolSize=5, maxPoolSize=5
2024-11-17T20:36:02,108 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/0b0b0ae8de03:0, corePoolSize=10, maxPoolSize=10
2024-11-17T20:36:02,108 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:36:02,109 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/0b0b0ae8de03:0, corePoolSize=2, maxPoolSize=2
2024-11-17T20:36:02,109 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:36:02,110 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731875792110
2024-11-17T20:36:02,110 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-17T20:36:02,110 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-17T20:36:02,110 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-17T20:36:02,110 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-17T20:36:02,110 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-17T20:36:02,110 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-17T20:36:02,111 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-17T20:36:02,111 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-17T20:36:02,111 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-17T20:36:02,112 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-17T20:36:02,112 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-17T20:36:02,112 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-17T20:36:02,112 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:36:02,112 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-17T20:36:02,116 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-17T20:36:02,116 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-17T20:36:02,117 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/0b0b0ae8de03:0:becomeActiveMaster-HFileCleaner.large.0-1731875762116,5,FailOnTimeoutGroup]
2024-11-17T20:36:02,117 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/0b0b0ae8de03:0:becomeActiveMaster-HFileCleaner.small.0-1731875762117,5,FailOnTimeoutGroup]
2024-11-17T20:36:02,117 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-17T20:36:02,117 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-17T20:36:02,117 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-17T20:36:02,118 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-17T20:36:02,123 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741831_1007 (size=1321)
2024-11-17T20:36:02,123 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46273 is added to blk_1073741831_1007 (size=1321)
2024-11-17T20:36:02,124 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-17T20:36:02,124 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6
2024-11-17T20:36:02,127 INFO  [RS:0;0b0b0ae8de03:35963 {}] regionserver.HRegionServer(746): ClusterId : a1b4dec0-8444-4d02-83de-c771e6b96f93
2024-11-17T20:36:02,128 DEBUG [RS:0;0b0b0ae8de03:35963 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-17T20:36:02,132 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46273 is added to blk_1073741832_1008 (size=32)
2024-11-17T20:36:02,133 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741832_1008 (size=32)
2024-11-17T20:36:02,133 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T20:36:02,134 DEBUG [RS:0;0b0b0ae8de03:35963 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-17T20:36:02,134 DEBUG [RS:0;0b0b0ae8de03:35963 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-17T20:36:02,135 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-17T20:36:02,136 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-17T20:36:02,136 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:36:02,137 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T20:36:02,137 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-17T20:36:02,138 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-17T20:36:02,138 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:36:02,139 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T20:36:02,139 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-17T20:36:02,140 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-17T20:36:02,140 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:36:02,140 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T20:36:02,140 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-17T20:36:02,141 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-17T20:36:02,142 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:36:02,142 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T20:36:02,142 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-17T20:36:02,143 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/data/hbase/meta/1588230740
2024-11-17T20:36:02,143 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/data/hbase/meta/1588230740
2024-11-17T20:36:02,144 DEBUG [RS:0;0b0b0ae8de03:35963 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-17T20:36:02,144 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-17T20:36:02,144 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-17T20:36:02,145 DEBUG [RS:0;0b0b0ae8de03:35963 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11843b2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0b0b0ae8de03/172.17.0.2:0
2024-11-17T20:36:02,145 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-17T20:36:02,146 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-17T20:36:02,148 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-17T20:36:02,148 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=835207, jitterRate=0.06202138960361481}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-17T20:36:02,149 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1731875762134Initializing all the Stores at 1731875762134Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731875762134Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731875762135 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731875762135Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731875762135Cleaning up temporary data from old regions at 1731875762144 (+9 ms)Region opened successfully at 1731875762149 (+5 ms)
2024-11-17T20:36:02,149 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-17T20:36:02,149 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-17T20:36:02,149 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-17T20:36:02,149 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-17T20:36:02,149 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-17T20:36:02,149 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-17T20:36:02,149 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731875762149Disabling compacts and flushes for region at 1731875762149Disabling writes for close at 1731875762149Writing region close event to WAL at 1731875762149Closed at 1731875762149
2024-11-17T20:36:02,151 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-17T20:36:02,151 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-17T20:36:02,151 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-17T20:36:02,152 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-17T20:36:02,153 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-17T20:36:02,157 DEBUG [RS:0;0b0b0ae8de03:35963 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;0b0b0ae8de03:35963
2024-11-17T20:36:02,157 INFO  [RS:0;0b0b0ae8de03:35963 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-17T20:36:02,157 INFO  [RS:0;0b0b0ae8de03:35963 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-17T20:36:02,157 DEBUG [RS:0;0b0b0ae8de03:35963 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-17T20:36:02,158 INFO  [RS:0;0b0b0ae8de03:35963 {}] regionserver.HRegionServer(2659): reportForDuty to master=0b0b0ae8de03,36429,1731875761121 with port=35963, startcode=1731875761298
2024-11-17T20:36:02,158 DEBUG [RS:0;0b0b0ae8de03:35963 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-17T20:36:02,160 INFO  [HMaster-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49291, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.4 (auth:SIMPLE), service=RegionServerStatusService
2024-11-17T20:36:02,161 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36429 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 0b0b0ae8de03,35963,1731875761298
2024-11-17T20:36:02,161 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36429 {}] master.ServerManager(517): Registering regionserver=0b0b0ae8de03,35963,1731875761298
2024-11-17T20:36:02,162 DEBUG [RS:0;0b0b0ae8de03:35963 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6
2024-11-17T20:36:02,162 DEBUG [RS:0;0b0b0ae8de03:35963 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:46233
2024-11-17T20:36:02,163 DEBUG [RS:0;0b0b0ae8de03:35963 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-17T20:36:02,175 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36429-0x1014a82e7730000, quorum=127.0.0.1:49481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-17T20:36:02,176 DEBUG [RS:0;0b0b0ae8de03:35963 {}] zookeeper.ZKUtil(111): regionserver:35963-0x1014a82e7730001, quorum=127.0.0.1:49481, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/0b0b0ae8de03,35963,1731875761298
2024-11-17T20:36:02,176 WARN  [RS:0;0b0b0ae8de03:35963 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-17T20:36:02,176 INFO  [RS:0;0b0b0ae8de03:35963 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-17T20:36:02,176 DEBUG [RS:0;0b0b0ae8de03:35963 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/WALs/0b0b0ae8de03,35963,1731875761298
2024-11-17T20:36:02,176 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [0b0b0ae8de03,35963,1731875761298]
2024-11-17T20:36:02,179 INFO  [RS:0;0b0b0ae8de03:35963 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-17T20:36:02,188 INFO  [RS:0;0b0b0ae8de03:35963 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-17T20:36:02,189 INFO  [RS:0;0b0b0ae8de03:35963 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-17T20:36:02,189 INFO  [RS:0;0b0b0ae8de03:35963 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-17T20:36:02,189 INFO  [RS:0;0b0b0ae8de03:35963 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-17T20:36:02,190 INFO  [RS:0;0b0b0ae8de03:35963 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-17T20:36:02,190 INFO  [RS:0;0b0b0ae8de03:35963 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-17T20:36:02,190 DEBUG [RS:0;0b0b0ae8de03:35963 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:36:02,190 DEBUG [RS:0;0b0b0ae8de03:35963 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:36:02,190 DEBUG [RS:0;0b0b0ae8de03:35963 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:36:02,190 DEBUG [RS:0;0b0b0ae8de03:35963 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:36:02,190 DEBUG [RS:0;0b0b0ae8de03:35963 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:36:02,191 DEBUG [RS:0;0b0b0ae8de03:35963 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/0b0b0ae8de03:0, corePoolSize=2, maxPoolSize=2
2024-11-17T20:36:02,191 DEBUG [RS:0;0b0b0ae8de03:35963 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:36:02,191 DEBUG [RS:0;0b0b0ae8de03:35963 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:36:02,191 DEBUG [RS:0;0b0b0ae8de03:35963 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:36:02,191 DEBUG [RS:0;0b0b0ae8de03:35963 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:36:02,191 DEBUG [RS:0;0b0b0ae8de03:35963 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:36:02,191 DEBUG [RS:0;0b0b0ae8de03:35963 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:36:02,191 DEBUG [RS:0;0b0b0ae8de03:35963 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/0b0b0ae8de03:0, corePoolSize=3, maxPoolSize=3
2024-11-17T20:36:02,191 DEBUG [RS:0;0b0b0ae8de03:35963 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0, corePoolSize=3, maxPoolSize=3
2024-11-17T20:36:02,193 INFO  [RS:0;0b0b0ae8de03:35963 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-17T20:36:02,193 INFO  [RS:0;0b0b0ae8de03:35963 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-17T20:36:02,193 INFO  [RS:0;0b0b0ae8de03:35963 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-17T20:36:02,193 INFO  [RS:0;0b0b0ae8de03:35963 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-17T20:36:02,193 INFO  [RS:0;0b0b0ae8de03:35963 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-17T20:36:02,193 INFO  [RS:0;0b0b0ae8de03:35963 {}] hbase.ChoreService(168): Chore ScheduledChore name=0b0b0ae8de03,35963,1731875761298-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-17T20:36:02,210 INFO  [RS:0;0b0b0ae8de03:35963 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-17T20:36:02,210 INFO  [RS:0;0b0b0ae8de03:35963 {}] hbase.ChoreService(168): Chore ScheduledChore name=0b0b0ae8de03,35963,1731875761298-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-17T20:36:02,210 INFO  [RS:0;0b0b0ae8de03:35963 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T20:36:02,210 INFO  [RS:0;0b0b0ae8de03:35963 {}] regionserver.Replication(171): 0b0b0ae8de03,35963,1731875761298 started
2024-11-17T20:36:02,227 INFO  [RS:0;0b0b0ae8de03:35963 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T20:36:02,227 INFO  [RS:0;0b0b0ae8de03:35963 {}] regionserver.HRegionServer(1482): Serving as 0b0b0ae8de03,35963,1731875761298, RpcServer on 0b0b0ae8de03/172.17.0.2:35963, sessionid=0x1014a82e7730001
2024-11-17T20:36:02,227 DEBUG [RS:0;0b0b0ae8de03:35963 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-17T20:36:02,227 DEBUG [RS:0;0b0b0ae8de03:35963 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 0b0b0ae8de03,35963,1731875761298
2024-11-17T20:36:02,227 DEBUG [RS:0;0b0b0ae8de03:35963 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0b0b0ae8de03,35963,1731875761298'
2024-11-17T20:36:02,227 DEBUG [RS:0;0b0b0ae8de03:35963 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-17T20:36:02,228 DEBUG [RS:0;0b0b0ae8de03:35963 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-17T20:36:02,228 DEBUG [RS:0;0b0b0ae8de03:35963 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-17T20:36:02,228 DEBUG [RS:0;0b0b0ae8de03:35963 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-17T20:36:02,228 DEBUG [RS:0;0b0b0ae8de03:35963 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 0b0b0ae8de03,35963,1731875761298
2024-11-17T20:36:02,228 DEBUG [RS:0;0b0b0ae8de03:35963 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0b0b0ae8de03,35963,1731875761298'
2024-11-17T20:36:02,228 DEBUG [RS:0;0b0b0ae8de03:35963 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-17T20:36:02,229 DEBUG [RS:0;0b0b0ae8de03:35963 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-17T20:36:02,229 DEBUG [RS:0;0b0b0ae8de03:35963 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-17T20:36:02,229 INFO  [RS:0;0b0b0ae8de03:35963 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-17T20:36:02,229 INFO  [RS:0;0b0b0ae8de03:35963 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-17T20:36:02,303 WARN  [0b0b0ae8de03:36429 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions.
2024-11-17T20:36:02,331 INFO  [RS:0;0b0b0ae8de03:35963 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0b0b0ae8de03%2C35963%2C1731875761298, suffix=, logDir=hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/WALs/0b0b0ae8de03,35963,1731875761298, archiveDir=hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/oldWALs, maxLogs=32
2024-11-17T20:36:02,332 INFO  [RS:0;0b0b0ae8de03:35963 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0b0b0ae8de03%2C35963%2C1731875761298.1731875762332
2024-11-17T20:36:02,342 INFO  [RS:0;0b0b0ae8de03:35963 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/WALs/0b0b0ae8de03,35963,1731875761298/0b0b0ae8de03%2C35963%2C1731875761298.1731875762332
2024-11-17T20:36:02,343 DEBUG [RS:0;0b0b0ae8de03:35963 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34383:34383),(127.0.0.1/127.0.0.1:38015:38015)]
2024-11-17T20:36:02,553 DEBUG [0b0b0ae8de03:36429 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-17T20:36:02,554 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=0b0b0ae8de03,35963,1731875761298
2024-11-17T20:36:02,556 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0b0b0ae8de03,35963,1731875761298, state=OPENING
2024-11-17T20:36:02,605 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-17T20:36:02,617 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35963-0x1014a82e7730001, quorum=127.0.0.1:49481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:36:02,617 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36429-0x1014a82e7730000, quorum=127.0.0.1:49481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:36:02,618 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-17T20:36:02,618 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-17T20:36:02,618 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-17T20:36:02,618 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=0b0b0ae8de03,35963,1731875761298}]
2024-11-17T20:36:02,762 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:02,772 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-17T20:36:02,774 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41043, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-17T20:36:02,778 INFO  [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-17T20:36:02,778 INFO  [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-17T20:36:02,778 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:02,780 INFO  [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0b0b0ae8de03%2C35963%2C1731875761298.meta, suffix=.meta, logDir=hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/WALs/0b0b0ae8de03,35963,1731875761298, archiveDir=hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/oldWALs, maxLogs=32
2024-11-17T20:36:02,780 INFO  [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 0b0b0ae8de03%2C35963%2C1731875761298.meta.1731875762780.meta
2024-11-17T20:36:02,785 INFO  [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/WALs/0b0b0ae8de03,35963,1731875761298/0b0b0ae8de03%2C35963%2C1731875761298.meta.1731875762780.meta
2024-11-17T20:36:02,786 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34383:34383),(127.0.0.1/127.0.0.1:38015:38015)]
2024-11-17T20:36:02,788 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-17T20:36:02,789 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-17T20:36:02,789 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-17T20:36:02,789 INFO  [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-17T20:36:02,789 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-17T20:36:02,789 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T20:36:02,789 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-17T20:36:02,789 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-17T20:36:02,790 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-17T20:36:02,791 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-17T20:36:02,791 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:36:02,791 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T20:36:02,792 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-17T20:36:02,792 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-17T20:36:02,792 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:36:02,793 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T20:36:02,793 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-17T20:36:02,793 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-17T20:36:02,793 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:36:02,794 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T20:36:02,794 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-17T20:36:02,794 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-17T20:36:02,794 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:36:02,795 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T20:36:02,795 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-17T20:36:02,795 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/data/hbase/meta/1588230740
2024-11-17T20:36:02,796 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/data/hbase/meta/1588230740
2024-11-17T20:36:02,798 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-17T20:36:02,798 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-17T20:36:02,798 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-17T20:36:02,799 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-17T20:36:02,800 INFO  [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=700588, jitterRate=-0.10915741324424744}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-17T20:36:02,800 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-17T20:36:02,801 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1731875762789Writing region info on filesystem at 1731875762789Initializing all the Stores at 1731875762790 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731875762790Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731875762790Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731875762790Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731875762790Cleaning up temporary data from old regions at 1731875762798 (+8 ms)Running coprocessor post-open hooks at 1731875762800 (+2 ms)Region opened successfully at 1731875762801 (+1 ms)
2024-11-17T20:36:02,802 INFO  [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731875762772
2024-11-17T20:36:02,804 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-17T20:36:02,804 INFO  [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-17T20:36:02,805 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=0b0b0ae8de03,35963,1731875761298
2024-11-17T20:36:02,806 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0b0b0ae8de03,35963,1731875761298, state=OPEN
2024-11-17T20:36:02,848 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36429-0x1014a82e7730000, quorum=127.0.0.1:49481, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-17T20:36:02,848 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35963-0x1014a82e7730001, quorum=127.0.0.1:49481, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-17T20:36:02,848 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=0b0b0ae8de03,35963,1731875761298
2024-11-17T20:36:02,849 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-17T20:36:02,849 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-17T20:36:02,852 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-17T20:36:02,852 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=0b0b0ae8de03,35963,1731875761298 in 230 msec
2024-11-17T20:36:02,855 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-17T20:36:02,855 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 701 msec
2024-11-17T20:36:02,855 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-17T20:36:02,855 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-17T20:36:02,857 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-17T20:36:02,857 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,35963,1731875761298, seqNum=-1]
2024-11-17T20:36:02,857 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-17T20:36:02,858 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-11-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47467, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-17T20:36:02,864 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 758 msec
2024-11-17T20:36:02,864 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731875762864, completionTime=-1
2024-11-17T20:36:02,864 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-17T20:36:02,864 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster...
2024-11-17T20:36:02,866 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1
2024-11-17T20:36:02,866 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731875822866
2024-11-17T20:36:02,866 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731875882866
2024-11-17T20:36:02,866 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 1 msec
2024-11-17T20:36:02,866 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0b0b0ae8de03,36429,1731875761121-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-17T20:36:02,867 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0b0b0ae8de03,36429,1731875761121-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T20:36:02,867 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0b0b0ae8de03,36429,1731875761121-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T20:36:02,867 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-0b0b0ae8de03:36429, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T20:36:02,867 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-17T20:36:02,867 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-17T20:36:02,869 DEBUG [master/0b0b0ae8de03:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-17T20:36:02,871 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.506sec
2024-11-17T20:36:02,871 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-17T20:36:02,871 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-17T20:36:02,871 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-17T20:36:02,871 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-17T20:36:02,872 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-17T20:36:02,872 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0b0b0ae8de03,36429,1731875761121-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-17T20:36:02,872 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0b0b0ae8de03,36429,1731875761121-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-17T20:36:02,875 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-17T20:36:02,875 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-17T20:36:02,875 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0b0b0ae8de03,36429,1731875761121-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T20:36:02,928 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70770496, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-17T20:36:02,928 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b0b0ae8de03,36429,-1 for getting cluster id
2024-11-17T20:36:02,928 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-17T20:36:02,929 DEBUG [HMaster-EventLoopGroup-10-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1b4dec0-8444-4d02-83de-c771e6b96f93'
2024-11-17T20:36:02,930 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-17T20:36:02,930 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1b4dec0-8444-4d02-83de-c771e6b96f93"

2024-11-17T20:36:02,930 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b8296aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-17T20:36:02,930 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b0b0ae8de03,36429,-1]
2024-11-17T20:36:02,930 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-17T20:36:02,931 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T20:36:02,932 INFO  [HMaster-EventLoopGroup-10-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57394, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-17T20:36:02,933 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4254f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-17T20:36:02,933 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-17T20:36:02,934 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,35963,1731875761298, seqNum=-1]
2024-11-17T20:36:02,934 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-17T20:36:02,934 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:02,936 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51658, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-17T20:36:02,938 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=0b0b0ae8de03,36429,1731875761121
2024-11-17T20:36:02,938 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T20:36:02,941 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-17T20:36:02,941 INFO  [Time-limited test {}] wal.TestLogRolling(320): Starting testLogRollOnPipelineRestart
2024-11-17T20:36:02,941 INFO  [Time-limited test {}] wal.TestLogRolling(323): Replication=2
2024-11-17T20:36:02,941 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-17T20:36:02,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncConnectionImpl(321): The fetched master address is 0b0b0ae8de03,36429,1731875761121
2024-11-17T20:36:02,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6edb6b43
2024-11-17T20:36:02,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-17T20:36:02,944 INFO  [HMaster-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57396, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-17T20:36:02,945 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36429 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-11-17T20:36:02,945 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36429 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-11-17T20:36:02,945 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36429 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRollOnPipelineRestart', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-17T20:36:02,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36429 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart
2024-11-17T20:36:02,948 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_PRE_OPERATION
2024-11-17T20:36:02,948 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:36:02,948 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36429 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRollOnPipelineRestart"
 procId is: 4
2024-11-17T20:36:02,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-17T20:36:02,949 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-17T20:36:02,956 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741835_1011 (size=395)
2024-11-17T20:36:02,956 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46273 is added to blk_1073741835_1011 (size=395)
2024-11-17T20:36:02,958 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 2e44545a7215dcd0d833f62680b5cba5, NAME => 'TestLogRolling-testLogRollOnPipelineRestart,,1731875762945.2e44545a7215dcd0d833f62680b5cba5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnPipelineRestart', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6
2024-11-17T20:36:02,964 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741836_1012 (size=78)
2024-11-17T20:36:02,964 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46273 is added to blk_1073741836_1012 (size=78)
2024-11-17T20:36:02,965 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnPipelineRestart,,1731875762945.2e44545a7215dcd0d833f62680b5cba5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T20:36:02,965 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1722): Closing 2e44545a7215dcd0d833f62680b5cba5, disabling compactions & flushes
2024-11-17T20:36:02,965 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1731875762945.2e44545a7215dcd0d833f62680b5cba5.
2024-11-17T20:36:02,965 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1731875762945.2e44545a7215dcd0d833f62680b5cba5.
2024-11-17T20:36:02,965 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1731875762945.2e44545a7215dcd0d833f62680b5cba5. after waiting 0 ms
2024-11-17T20:36:02,965 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1731875762945.2e44545a7215dcd0d833f62680b5cba5.
2024-11-17T20:36:02,965 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnPipelineRestart,,1731875762945.2e44545a7215dcd0d833f62680b5cba5.
2024-11-17T20:36:02,965 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1676): Region close journal for 2e44545a7215dcd0d833f62680b5cba5:
Waiting for close lock at 1731875762965Disabling compacts and flushes for region at 1731875762965Disabling writes for close at 1731875762965Writing region close event to WAL at 1731875762965Closed at 1731875762965
2024-11-17T20:36:02,966 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_ADD_TO_META
2024-11-17T20:36:02,966 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnPipelineRestart,,1731875762945.2e44545a7215dcd0d833f62680b5cba5.","families":{"info":[{"qualifier":"regioninfo","vlen":77,"tag":[],"timestamp":"1731875762966"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731875762966"}]},"ts":"1731875762966"}
2024-11-17T20:36:02,969 INFO  [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-17T20:36:02,970 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-17T20:36:02,970 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnPipelineRestart","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731875762970"}]},"ts":"1731875762970"}
2024-11-17T20:36:02,972 INFO  [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnPipelineRestart, state=ENABLING in hbase:meta
2024-11-17T20:36:02,973 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=2e44545a7215dcd0d833f62680b5cba5, ASSIGN}]
2024-11-17T20:36:02,974 INFO  [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=2e44545a7215dcd0d833f62680b5cba5, ASSIGN
2024-11-17T20:36:02,975 INFO  [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=2e44545a7215dcd0d833f62680b5cba5, ASSIGN; state=OFFLINE, location=0b0b0ae8de03,35963,1731875761298; forceNewPlan=false, retain=false
2024-11-17T20:36:03,126 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=2e44545a7215dcd0d833f62680b5cba5, regionState=OPENING, regionLocation=0b0b0ae8de03,35963,1731875761298
2024-11-17T20:36:03,133 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-11-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=2e44545a7215dcd0d833f62680b5cba5, ASSIGN because future has completed
2024-11-17T20:36:03,134 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2e44545a7215dcd0d833f62680b5cba5, server=0b0b0ae8de03,35963,1731875761298}]
2024-11-17T20:36:03,295 INFO  [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRollOnPipelineRestart,,1731875762945.2e44545a7215dcd0d833f62680b5cba5.
2024-11-17T20:36:03,296 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 2e44545a7215dcd0d833f62680b5cba5, NAME => 'TestLogRolling-testLogRollOnPipelineRestart,,1731875762945.2e44545a7215dcd0d833f62680b5cba5.', STARTKEY => '', ENDKEY => ''}
2024-11-17T20:36:03,296 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnPipelineRestart 2e44545a7215dcd0d833f62680b5cba5
2024-11-17T20:36:03,297 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnPipelineRestart,,1731875762945.2e44545a7215dcd0d833f62680b5cba5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T20:36:03,297 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 2e44545a7215dcd0d833f62680b5cba5
2024-11-17T20:36:03,297 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 2e44545a7215dcd0d833f62680b5cba5
2024-11-17T20:36:03,300 INFO  [StoreOpener-2e44545a7215dcd0d833f62680b5cba5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 2e44545a7215dcd0d833f62680b5cba5 
2024-11-17T20:36:03,301 INFO  [StoreOpener-2e44545a7215dcd0d833f62680b5cba5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2e44545a7215dcd0d833f62680b5cba5 columnFamilyName info
2024-11-17T20:36:03,301 DEBUG [StoreOpener-2e44545a7215dcd0d833f62680b5cba5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:36:03,302 INFO  [StoreOpener-2e44545a7215dcd0d833f62680b5cba5-1 {}] regionserver.HStore(327): Store=2e44545a7215dcd0d833f62680b5cba5/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-17T20:36:03,302 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 2e44545a7215dcd0d833f62680b5cba5
2024-11-17T20:36:03,303 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/data/default/TestLogRolling-testLogRollOnPipelineRestart/2e44545a7215dcd0d833f62680b5cba5
2024-11-17T20:36:03,303 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/data/default/TestLogRolling-testLogRollOnPipelineRestart/2e44545a7215dcd0d833f62680b5cba5
2024-11-17T20:36:03,304 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 2e44545a7215dcd0d833f62680b5cba5
2024-11-17T20:36:03,304 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 2e44545a7215dcd0d833f62680b5cba5
2024-11-17T20:36:03,306 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 2e44545a7215dcd0d833f62680b5cba5
2024-11-17T20:36:03,308 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/data/default/TestLogRolling-testLogRollOnPipelineRestart/2e44545a7215dcd0d833f62680b5cba5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-17T20:36:03,308 INFO  [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 2e44545a7215dcd0d833f62680b5cba5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=816152, jitterRate=0.037792086601257324}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-17T20:36:03,308 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2e44545a7215dcd0d833f62680b5cba5
2024-11-17T20:36:03,309 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 2e44545a7215dcd0d833f62680b5cba5:
Running coprocessor pre-open hook at 1731875763297Writing region info on filesystem at 1731875763297Initializing all the Stores at 1731875763299 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731875763299Cleaning up temporary data from old regions at 1731875763304 (+5 ms)Running coprocessor post-open hooks at 1731875763308 (+4 ms)Region opened successfully at 1731875763309 (+1 ms)
2024-11-17T20:36:03,310 INFO  [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRollOnPipelineRestart,,1731875762945.2e44545a7215dcd0d833f62680b5cba5., pid=6, masterSystemTime=1731875763290
2024-11-17T20:36:03,312 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRollOnPipelineRestart,,1731875762945.2e44545a7215dcd0d833f62680b5cba5.
2024-11-17T20:36:03,313 INFO  [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRollOnPipelineRestart,,1731875762945.2e44545a7215dcd0d833f62680b5cba5.
2024-11-17T20:36:03,313 INFO  [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=2e44545a7215dcd0d833f62680b5cba5, regionState=OPEN, openSeqNum=2, regionLocation=0b0b0ae8de03,35963,1731875761298
2024-11-17T20:36:03,316 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-11-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2e44545a7215dcd0d833f62680b5cba5, server=0b0b0ae8de03,35963,1731875761298 because future has completed
2024-11-17T20:36:03,320 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-17T20:36:03,320 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 2e44545a7215dcd0d833f62680b5cba5, server=0b0b0ae8de03,35963,1731875761298 in 183 msec
2024-11-17T20:36:03,322 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-17T20:36:03,322 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=2e44545a7215dcd0d833f62680b5cba5, ASSIGN in 347 msec
2024-11-17T20:36:03,323 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-17T20:36:03,323 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnPipelineRestart","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731875763323"}]},"ts":"1731875763323"}
2024-11-17T20:36:03,325 INFO  [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnPipelineRestart, state=ENABLED in hbase:meta
2024-11-17T20:36:03,326 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_POST_OPERATION
2024-11-17T20:36:03,328 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart in 381 msec
2024-11-17T20:36:03,762 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:03,779 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:03,803 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-17T20:36:03,803 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-11-17T20:36:03,805 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart
2024-11-17T20:36:03,805 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart Metrics about Tables on a single HBase RegionServer
2024-11-17T20:36:03,806 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-17T20:36:03,806 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-11-17T20:36:03,806 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-11-17T20:36:03,806 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer
2024-11-17T20:36:03,935 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:04,763 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:04,780 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:04,936 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:05,764 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:05,780 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:05,937 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:06,765 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:06,781 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:06,937 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:07,765 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:07,782 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:07,938 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:08,291 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-17T20:36:08,314 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:36:08,314 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:36:08,314 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:36:08,315 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:36:08,315 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:36:08,316 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:36:08,321 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:36:08,321 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:36:08,322 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:36:08,325 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:36:08,331 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-17T20:36:08,332 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnPipelineRestart'
2024-11-17T20:36:08,766 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:08,782 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:08,939 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:09,767 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:09,783 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:09,940 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:10,767 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:10,784 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:10,940 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:11,768 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:11,785 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:11,941 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:12,769 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:12,785 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:12,942 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:13,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-17T20:36:13,028 INFO  [RPCClient-NioEventLoopGroup-4-15 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnPipelineRestart completed
2024-11-17T20:36:13,028 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRollOnPipelineRestart,, stopping at row=TestLogRolling-testLogRollOnPipelineRestart ,, for max=2147483647 with caching=100
2024-11-17T20:36:13,031 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRollOnPipelineRestart
2024-11-17T20:36:13,032 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRollOnPipelineRestart,,1731875762945.2e44545a7215dcd0d833f62680b5cba5.
2024-11-17T20:36:13,036 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRollOnPipelineRestart', row='row1002', locateType=CURRENT is [region=TestLogRolling-testLogRollOnPipelineRestart,,1731875762945.2e44545a7215dcd0d833f62680b5cba5., hostname=0b0b0ae8de03,35963,1731875761298, seqNum=2]
2024-11-17T20:36:13,770 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:13,786 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:13,943 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:14,770 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:14,787 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:14,944 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:15,039 INFO  [Time-limited test {}] wal.TestLogRolling(360): log.getCurrentFileName()): hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/WALs/0b0b0ae8de03,35963,1731875761298/0b0b0ae8de03%2C35963%2C1731875761298.1731875762332
2024-11-17T20:36:15,039 WARN  [ResponseProcessor for block BP-1977883270-172.17.0.2-1731875758461:blk_1073741830_1006 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1977883270-172.17.0.2-1731875758461:blk_1073741830_1006
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:36:15,039 WARN  [ResponseProcessor for block BP-1977883270-172.17.0.2-1731875758461:blk_1073741834_1010 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1977883270-172.17.0.2-1731875758461:blk_1073741834_1010
java.io.IOException: Bad response ERROR for BP-1977883270-172.17.0.2-1731875758461:blk_1073741834_1010 from datanode DatanodeInfoWithStorage[127.0.0.1:46273,DS-658f96ea-3d36-4123-8e44-dc53dc482242,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:36:15,040 WARN  [DataStreamer for file /user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/WALs/0b0b0ae8de03,35963,1731875761298/0b0b0ae8de03%2C35963%2C1731875761298.meta.1731875762780.meta block BP-1977883270-172.17.0.2-1731875758461:blk_1073741834_1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1977883270-172.17.0.2-1731875758461:blk_1073741834_1010 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45599,DS-5091058c-e4c0-4649-ad99-02c88df21742,DISK], DatanodeInfoWithStorage[127.0.0.1:46273,DS-658f96ea-3d36-4123-8e44-dc53dc482242,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:46273,DS-658f96ea-3d36-4123-8e44-dc53dc482242,DISK]) is bad.
2024-11-17T20:36:15,040 WARN  [ResponseProcessor for block BP-1977883270-172.17.0.2-1731875758461:blk_1073741833_1009 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1977883270-172.17.0.2-1731875758461:blk_1073741833_1009
java.io.IOException: Bad response ERROR for BP-1977883270-172.17.0.2-1731875758461:blk_1073741833_1009 from datanode DatanodeInfoWithStorage[127.0.0.1:46273,DS-658f96ea-3d36-4123-8e44-dc53dc482242,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:36:15,040 WARN  [DataStreamer for file /user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/WALs/0b0b0ae8de03,35963,1731875761298/0b0b0ae8de03%2C35963%2C1731875761298.1731875762332 block BP-1977883270-172.17.0.2-1731875758461:blk_1073741833_1009 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1977883270-172.17.0.2-1731875758461:blk_1073741833_1009 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45599,DS-5091058c-e4c0-4649-ad99-02c88df21742,DISK], DatanodeInfoWithStorage[127.0.0.1:46273,DS-658f96ea-3d36-4123-8e44-dc53dc482242,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:46273,DS-658f96ea-3d36-4123-8e44-dc53dc482242,DISK]) is bad.
2024-11-17T20:36:15,040 WARN  [PacketResponder: BP-1977883270-172.17.0.2-1731875758461:blk_1073741834_1010, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:46273] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:36:15,040 WARN  [PacketResponder: BP-1977883270-172.17.0.2-1731875758461:blk_1073741833_1009, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:46273] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:36:15,041 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1220123956_22 at /127.0.0.1:35008 [Receiving block BP-1977883270-172.17.0.2-1731875758461:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:45599:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:35008 dst: /127.0.0.1:45599
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:36:15,041 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1220123956_22 at /127.0.0.1:35020 [Receiving block BP-1977883270-172.17.0.2-1731875758461:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:45599:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:35020 dst: /127.0.0.1:45599
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:36:15,041 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1220123956_22 at /127.0.0.1:43422 [Receiving block BP-1977883270-172.17.0.2-1731875758461:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:46273:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:43422 dst: /127.0.0.1:46273
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:36:15,041 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1220123956_22 at /127.0.0.1:43418 [Receiving block BP-1977883270-172.17.0.2-1731875758461:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:46273:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:43418 dst: /127.0.0.1:46273
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:36:15,041 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1765916636_22 at /127.0.0.1:50096 [Receiving block BP-1977883270-172.17.0.2-1731875758461:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:46273:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:50096 dst: /127.0.0.1:46273
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:36:15,041 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1765916636_22 at /127.0.0.1:33036 [Receiving block BP-1977883270-172.17.0.2-1731875758461:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:45599:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:33036 dst: /127.0.0.1:45599
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:36:15,042 WARN  [DataStreamer for file /user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/MasterData/WALs/0b0b0ae8de03,36429,1731875761121/0b0b0ae8de03%2C36429%2C1731875761121.1731875761855 block BP-1977883270-172.17.0.2-1731875758461:blk_1073741830_1006 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1977883270-172.17.0.2-1731875758461:blk_1073741830_1006 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46273,DS-658f96ea-3d36-4123-8e44-dc53dc482242,DISK], DatanodeInfoWithStorage[127.0.0.1:45599,DS-5091058c-e4c0-4649-ad99-02c88df21742,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:46273,DS-658f96ea-3d36-4123-8e44-dc53dc482242,DISK]) is bad.
2024-11-17T20:36:15,089 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@32ef6c57{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T20:36:15,089 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3c9f0fbb{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-17T20:36:15,089 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-17T20:36:15,090 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@68c19d09{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-17T20:36:15,090 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@40375d59{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d2981510-59e6-ff69-efd1-5906a8a92785/hadoop.log.dir/,STOPPED}
2024-11-17T20:36:15,091 WARN  [BP-1977883270-172.17.0.2-1731875758461 heartbeating to localhost/127.0.0.1:46233 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-17T20:36:15,091 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-17T20:36:15,091 WARN  [BP-1977883270-172.17.0.2-1731875758461 heartbeating to localhost/127.0.0.1:46233 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1977883270-172.17.0.2-1731875758461 (Datanode Uuid 35b41824-7467-4b8c-81a8-a7878b27f81a) service to localhost/127.0.0.1:46233
2024-11-17T20:36:15,091 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-17T20:36:15,091 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d2981510-59e6-ff69-efd1-5906a8a92785/cluster_a0b2892d-6b1d-843d-7c12-ddaf29516f86/data/data3/current/BP-1977883270-172.17.0.2-1731875758461 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T20:36:15,092 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d2981510-59e6-ff69-efd1-5906a8a92785/cluster_a0b2892d-6b1d-843d-7c12-ddaf29516f86/data/data4/current/BP-1977883270-172.17.0.2-1731875758461 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T20:36:15,092 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-17T20:36:15,100 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-17T20:36:15,104 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-17T20:36:15,105 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-17T20:36:15,105 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-17T20:36:15,105 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-17T20:36:15,106 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5f1a012{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d2981510-59e6-ff69-efd1-5906a8a92785/hadoop.log.dir/,AVAILABLE}
2024-11-17T20:36:15,106 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@117b424d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-17T20:36:15,211 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2647e756{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d2981510-59e6-ff69-efd1-5906a8a92785/java.io.tmpdir/jetty-localhost-38883-hadoop-hdfs-3_4_1-tests_jar-_-any-9728665761346527126/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T20:36:15,211 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@443f9e40{HTTP/1.1, (http/1.1)}{localhost:38883}
2024-11-17T20:36:15,211 INFO  [Time-limited test {}] server.Server(415): Started @178497ms
2024-11-17T20:36:15,212 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-17T20:36:15,232 WARN  [ResponseProcessor for block BP-1977883270-172.17.0.2-1731875758461:blk_1073741833_1014 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1977883270-172.17.0.2-1731875758461:blk_1073741833_1014
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:36:15,232 WARN  [ResponseProcessor for block BP-1977883270-172.17.0.2-1731875758461:blk_1073741830_1015 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1977883270-172.17.0.2-1731875758461:blk_1073741830_1015
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:36:15,232 WARN  [ResponseProcessor for block BP-1977883270-172.17.0.2-1731875758461:blk_1073741834_1013 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1977883270-172.17.0.2-1731875758461:blk_1073741834_1013
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:36:15,232 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1220123956_22 at /127.0.0.1:49264 [Receiving block BP-1977883270-172.17.0.2-1731875758461:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:45599:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:49264 dst: /127.0.0.1:45599
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:36:15,232 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1220123956_22 at /127.0.0.1:49266 [Receiving block BP-1977883270-172.17.0.2-1731875758461:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:45599:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:49266 dst: /127.0.0.1:45599
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:36:15,232 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1765916636_22 at /127.0.0.1:49288 [Receiving block BP-1977883270-172.17.0.2-1731875758461:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:45599:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:49288 dst: /127.0.0.1:45599
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:36:15,234 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@68f9eaf6{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T20:36:15,234 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6b8d1f8b{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-17T20:36:15,234 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-17T20:36:15,234 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@31e2f2e1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-17T20:36:15,234 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@667c8bc{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d2981510-59e6-ff69-efd1-5906a8a92785/hadoop.log.dir/,STOPPED}
2024-11-17T20:36:15,235 WARN  [BP-1977883270-172.17.0.2-1731875758461 heartbeating to localhost/127.0.0.1:46233 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-17T20:36:15,235 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-17T20:36:15,235 WARN  [BP-1977883270-172.17.0.2-1731875758461 heartbeating to localhost/127.0.0.1:46233 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1977883270-172.17.0.2-1731875758461 (Datanode Uuid ba381e1e-86b5-491b-b28d-cf1370bb3873) service to localhost/127.0.0.1:46233
2024-11-17T20:36:15,235 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-17T20:36:15,236 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d2981510-59e6-ff69-efd1-5906a8a92785/cluster_a0b2892d-6b1d-843d-7c12-ddaf29516f86/data/data1/current/BP-1977883270-172.17.0.2-1731875758461 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T20:36:15,236 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d2981510-59e6-ff69-efd1-5906a8a92785/cluster_a0b2892d-6b1d-843d-7c12-ddaf29516f86/data/data2/current/BP-1977883270-172.17.0.2-1731875758461 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T20:36:15,236 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-17T20:36:15,251 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-17T20:36:15,254 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-17T20:36:15,255 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-17T20:36:15,255 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-17T20:36:15,255 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-17T20:36:15,255 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5d802677{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d2981510-59e6-ff69-efd1-5906a8a92785/hadoop.log.dir/,AVAILABLE}
2024-11-17T20:36:15,256 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7b559376{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-17T20:36:15,362 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@929fb8a{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d2981510-59e6-ff69-efd1-5906a8a92785/java.io.tmpdir/jetty-localhost-33957-hadoop-hdfs-3_4_1-tests_jar-_-any-4126290675957859351/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T20:36:15,362 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@39f8899d{HTTP/1.1, (http/1.1)}{localhost:33957}
2024-11-17T20:36:15,362 INFO  [Time-limited test {}] server.Server(415): Started @178648ms
2024-11-17T20:36:15,363 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-17T20:36:15,771 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:15,787 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:15,810 WARN  [Thread-1343 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-17T20:36:15,814 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xda78a78048daa6d1 with lease ID 0xfece477006bdaacc: from storage DS-658f96ea-3d36-4123-8e44-dc53dc482242 node DatanodeRegistration(127.0.0.1:36439, datanodeUuid=35b41824-7467-4b8c-81a8-a7878b27f81a, infoPort=44059, infoSecurePort=0, ipcPort=45183, storageInfo=lv=-57;cid=testClusterID;nsid=566398173;c=1731875758461), blocks: 6, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-17T20:36:15,814 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xda78a78048daa6d1 with lease ID 0xfece477006bdaacc: from storage DS-314bf586-6b8c-4cfe-8550-7b7d9a017aef node DatanodeRegistration(127.0.0.1:36439, datanodeUuid=35b41824-7467-4b8c-81a8-a7878b27f81a, infoPort=44059, infoSecurePort=0, ipcPort=45183, storageInfo=lv=-57;cid=testClusterID;nsid=566398173;c=1731875758461), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-17T20:36:15,944 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:15,982 WARN  [Thread-1363 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-17T20:36:15,984 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x56a904f5c0a6bd80 with lease ID 0xfece477006bdaacd: from storage DS-5091058c-e4c0-4649-ad99-02c88df21742 node DatanodeRegistration(127.0.0.1:43287, datanodeUuid=ba381e1e-86b5-491b-b28d-cf1370bb3873, infoPort=33095, infoSecurePort=0, ipcPort=41151, storageInfo=lv=-57;cid=testClusterID;nsid=566398173;c=1731875758461), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-17T20:36:15,984 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x56a904f5c0a6bd80 with lease ID 0xfece477006bdaacd: from storage DS-06de2f39-3030-45da-be94-2451d740ea4b node DatanodeRegistration(127.0.0.1:43287, datanodeUuid=ba381e1e-86b5-491b-b28d-cf1370bb3873, infoPort=33095, infoSecurePort=0, ipcPort=41151, storageInfo=lv=-57;cid=testClusterID;nsid=566398173;c=1731875758461), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-17T20:36:16,386 INFO  [Time-limited test {}] wal.TestLogRolling(372): Data Nodes restarted
2024-11-17T20:36:16,388 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1002
2024-11-17T20:36:16,389 ERROR [FSHLog-0-hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6-prefix:0b0b0ae8de03,35963,1731875761298 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45599,DS-5091058c-e4c0-4649-ad99-02c88df21742,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:36:16,389 WARN  [FSHLog-0-hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6-prefix:0b0b0ae8de03,35963,1731875761298 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45599,DS-5091058c-e4c0-4649-ad99-02c88df21742,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:36:16,390 DEBUG [regionserver/0b0b0ae8de03:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 0b0b0ae8de03%2C35963%2C1731875761298:(num 1731875762332) roll requested
2024-11-17T20:36:16,390 INFO  [regionserver/0b0b0ae8de03:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0b0b0ae8de03%2C35963%2C1731875761298.1731875776390
2024-11-17T20:36:16,395 DEBUG [regionserver/0b0b0ae8de03:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/WALs/0b0b0ae8de03,35963,1731875761298/0b0b0ae8de03%2C35963%2C1731875761298.1731875762332 newFile=hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/WALs/0b0b0ae8de03,35963,1731875761298/0b0b0ae8de03%2C35963%2C1731875761298.1731875776390
2024-11-17T20:36:16,395 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:36:16,395 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:36:16,395 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:36:16,396 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:36:16,396 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:36:16,396 INFO  [regionserver/0b0b0ae8de03:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/WALs/0b0b0ae8de03,35963,1731875761298/0b0b0ae8de03%2C35963%2C1731875761298.1731875762332 with entries=2, filesize=1.59 KB; new WAL /user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/WALs/0b0b0ae8de03,35963,1731875761298/0b0b0ae8de03%2C35963%2C1731875761298.1731875776390
2024-11-17T20:36:16,396 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45599,DS-5091058c-e4c0-4649-ad99-02c88df21742,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:36:16,396 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45599,DS-5091058c-e4c0-4649-ad99-02c88df21742,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:36:16,396 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/WALs/0b0b0ae8de03,35963,1731875761298/0b0b0ae8de03%2C35963%2C1731875761298.1731875762332
2024-11-17T20:36:16,396 DEBUG [regionserver/0b0b0ae8de03:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44059:44059),(127.0.0.1/127.0.0.1:33095:33095)]
2024-11-17T20:36:16,397 DEBUG [regionserver/0b0b0ae8de03:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/WALs/0b0b0ae8de03,35963,1731875761298/0b0b0ae8de03%2C35963%2C1731875761298.1731875762332 is not closed yet, will try archiving it next time
2024-11-17T20:36:16,397 WARN  [IPC Server handler 0 on default port 46233 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/WALs/0b0b0ae8de03,35963,1731875761298/0b0b0ae8de03%2C35963%2C1731875761298.1731875762332 has not been closed. Lease recovery is in progress. RecoveryId = 1017 for block blk_1073741833_1014
2024-11-17T20:36:16,397 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/WALs/0b0b0ae8de03,35963,1731875761298/0b0b0ae8de03%2C35963%2C1731875761298.1731875762332 after 1ms
2024-11-17T20:36:16,772 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:16,788 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:16,945 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:17,772 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:17,788 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:17,813 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741833_1014: GenerationStamp not matched, existing replica is blk_1073741833_1009
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-11-17T20:36:17,946 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:18,402 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1003
2024-11-17T20:36:18,773 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:18,789 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:18,946 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:19,774 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:19,790 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:19,947 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:20,398 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/WALs/0b0b0ae8de03,35963,1731875761298/0b0b0ae8de03%2C35963%2C1731875761298.1731875762332 after 4002ms
2024-11-17T20:36:20,406 WARN  [ResponseProcessor for block BP-1977883270-172.17.0.2-1731875758461:blk_1073741837_1016 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1977883270-172.17.0.2-1731875758461:blk_1073741837_1016
java.io.IOException: Bad response ERROR for BP-1977883270-172.17.0.2-1731875758461:blk_1073741837_1016 from datanode DatanodeInfoWithStorage[127.0.0.1:43287,DS-5091058c-e4c0-4649-ad99-02c88df21742,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:36:20,406 WARN  [DataStreamer for file /user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/WALs/0b0b0ae8de03,35963,1731875761298/0b0b0ae8de03%2C35963%2C1731875761298.1731875776390 block BP-1977883270-172.17.0.2-1731875758461:blk_1073741837_1016 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1977883270-172.17.0.2-1731875758461:blk_1073741837_1016 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36439,DS-658f96ea-3d36-4123-8e44-dc53dc482242,DISK], DatanodeInfoWithStorage[127.0.0.1:43287,DS-5091058c-e4c0-4649-ad99-02c88df21742,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:43287,DS-5091058c-e4c0-4649-ad99-02c88df21742,DISK]) is bad.
2024-11-17T20:36:20,406 WARN  [PacketResponder: BP-1977883270-172.17.0.2-1731875758461:blk_1073741837_1016, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:43287] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:36:20,407 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1220123956_22 at /127.0.0.1:57022 [Receiving block BP-1977883270-172.17.0.2-1731875758461:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:36439:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:57022 dst: /127.0.0.1:36439
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:36:20,407 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1220123956_22 at /127.0.0.1:58456 [Receiving block BP-1977883270-172.17.0.2-1731875758461:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:43287:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:58456 dst: /127.0.0.1:43287
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:36:20,468 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@929fb8a{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T20:36:20,468 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@39f8899d{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-17T20:36:20,468 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-17T20:36:20,468 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7b559376{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-17T20:36:20,468 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5d802677{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d2981510-59e6-ff69-efd1-5906a8a92785/hadoop.log.dir/,STOPPED}
2024-11-17T20:36:20,470 WARN  [BP-1977883270-172.17.0.2-1731875758461 heartbeating to localhost/127.0.0.1:46233 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-17T20:36:20,470 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-17T20:36:20,470 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-17T20:36:20,470 WARN  [BP-1977883270-172.17.0.2-1731875758461 heartbeating to localhost/127.0.0.1:46233 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1977883270-172.17.0.2-1731875758461 (Datanode Uuid ba381e1e-86b5-491b-b28d-cf1370bb3873) service to localhost/127.0.0.1:46233
2024-11-17T20:36:20,471 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d2981510-59e6-ff69-efd1-5906a8a92785/cluster_a0b2892d-6b1d-843d-7c12-ddaf29516f86/data/data1/current/BP-1977883270-172.17.0.2-1731875758461 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T20:36:20,471 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d2981510-59e6-ff69-efd1-5906a8a92785/cluster_a0b2892d-6b1d-843d-7c12-ddaf29516f86/data/data2/current/BP-1977883270-172.17.0.2-1731875758461 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T20:36:20,471 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-17T20:36:20,478 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-17T20:36:20,482 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-17T20:36:20,483 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-17T20:36:20,483 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-17T20:36:20,483 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-17T20:36:20,484 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4a79eeb5{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d2981510-59e6-ff69-efd1-5906a8a92785/hadoop.log.dir/,AVAILABLE}
2024-11-17T20:36:20,484 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6d237fac{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-17T20:36:20,595 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@48919a97{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d2981510-59e6-ff69-efd1-5906a8a92785/java.io.tmpdir/jetty-localhost-46879-hadoop-hdfs-3_4_1-tests_jar-_-any-12843554371446537480/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T20:36:20,595 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4778d192{HTTP/1.1, (http/1.1)}{localhost:46879}
2024-11-17T20:36:20,595 INFO  [Time-limited test {}] server.Server(415): Started @183881ms
2024-11-17T20:36:20,596 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-17T20:36:20,629 WARN  [ResponseProcessor for block BP-1977883270-172.17.0.2-1731875758461:blk_1073741837_1018 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1977883270-172.17.0.2-1731875758461:blk_1073741837_1018
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:36:20,629 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1220123956_22 at /127.0.0.1:57044 [Receiving block BP-1977883270-172.17.0.2-1731875758461:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:36439:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:57044 dst: /127.0.0.1:36439
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:36:20,641 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2647e756{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T20:36:20,642 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@443f9e40{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-17T20:36:20,642 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-17T20:36:20,642 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@117b424d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-17T20:36:20,642 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5f1a012{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d2981510-59e6-ff69-efd1-5906a8a92785/hadoop.log.dir/,STOPPED}
2024-11-17T20:36:20,643 WARN  [BP-1977883270-172.17.0.2-1731875758461 heartbeating to localhost/127.0.0.1:46233 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-17T20:36:20,643 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-17T20:36:20,643 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-17T20:36:20,643 WARN  [BP-1977883270-172.17.0.2-1731875758461 heartbeating to localhost/127.0.0.1:46233 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1977883270-172.17.0.2-1731875758461 (Datanode Uuid 35b41824-7467-4b8c-81a8-a7878b27f81a) service to localhost/127.0.0.1:46233
2024-11-17T20:36:20,644 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d2981510-59e6-ff69-efd1-5906a8a92785/cluster_a0b2892d-6b1d-843d-7c12-ddaf29516f86/data/data3/current/BP-1977883270-172.17.0.2-1731875758461 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T20:36:20,644 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d2981510-59e6-ff69-efd1-5906a8a92785/cluster_a0b2892d-6b1d-843d-7c12-ddaf29516f86/data/data4/current/BP-1977883270-172.17.0.2-1731875758461 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T20:36:20,645 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-17T20:36:20,659 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-17T20:36:20,662 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-17T20:36:20,676 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-17T20:36:20,676 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-17T20:36:20,676 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-17T20:36:20,677 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@160f7ba9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d2981510-59e6-ff69-efd1-5906a8a92785/hadoop.log.dir/,AVAILABLE}
2024-11-17T20:36:20,677 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7fc558ce{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-17T20:36:20,774 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:20,790 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@39e17fd3{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d2981510-59e6-ff69-efd1-5906a8a92785/java.io.tmpdir/jetty-localhost-40189-hadoop-hdfs-3_4_1-tests_jar-_-any-5830293372702496996/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T20:36:20,790 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:20,791 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5b16a6e2{HTTP/1.1, (http/1.1)}{localhost:40189}
2024-11-17T20:36:20,791 INFO  [Time-limited test {}] server.Server(415): Started @184077ms
2024-11-17T20:36:20,792 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-17T20:36:20,948 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:21,122 WARN  [Thread-1417 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-17T20:36:21,124 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbd05c967573420ec with lease ID 0xfece477006bdaace: from storage DS-5091058c-e4c0-4649-ad99-02c88df21742 node DatanodeRegistration(127.0.0.1:34593, datanodeUuid=ba381e1e-86b5-491b-b28d-cf1370bb3873, infoPort=35499, infoSecurePort=0, ipcPort=41541, storageInfo=lv=-57;cid=testClusterID;nsid=566398173;c=1731875758461), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-17T20:36:21,124 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbd05c967573420ec with lease ID 0xfece477006bdaace: from storage DS-06de2f39-3030-45da-be94-2451d740ea4b node DatanodeRegistration(127.0.0.1:34593, datanodeUuid=ba381e1e-86b5-491b-b28d-cf1370bb3873, infoPort=35499, infoSecurePort=0, ipcPort=41541, storageInfo=lv=-57;cid=testClusterID;nsid=566398173;c=1731875758461), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-17T20:36:21,344 WARN  [Thread-1437 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-17T20:36:21,346 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf033f8633c2c55d3 with lease ID 0xfece477006bdaacf: from storage DS-658f96ea-3d36-4123-8e44-dc53dc482242 node DatanodeRegistration(127.0.0.1:38053, datanodeUuid=35b41824-7467-4b8c-81a8-a7878b27f81a, infoPort=43029, infoSecurePort=0, ipcPort=40937, storageInfo=lv=-57;cid=testClusterID;nsid=566398173;c=1731875758461), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-17T20:36:21,347 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf033f8633c2c55d3 with lease ID 0xfece477006bdaacf: from storage DS-314bf586-6b8c-4cfe-8550-7b7d9a017aef node DatanodeRegistration(127.0.0.1:38053, datanodeUuid=35b41824-7467-4b8c-81a8-a7878b27f81a, infoPort=43029, infoSecurePort=0, ipcPort=40937, storageInfo=lv=-57;cid=testClusterID;nsid=566398173;c=1731875758461), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-17T20:36:21,775 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:21,791 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:21,812 INFO  [Time-limited test {}] wal.TestLogRolling(389): Data Nodes restarted
2024-11-17T20:36:21,815 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1004
2024-11-17T20:36:21,816 ERROR [FSHLog-0-hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6-prefix:0b0b0ae8de03,35963,1731875761298 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36439,DS-658f96ea-3d36-4123-8e44-dc53dc482242,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:36:21,817 WARN  [FSHLog-0-hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6-prefix:0b0b0ae8de03,35963,1731875761298 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36439,DS-658f96ea-3d36-4123-8e44-dc53dc482242,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:36:21,817 DEBUG [regionserver/0b0b0ae8de03:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 0b0b0ae8de03%2C35963%2C1731875761298:(num 1731875776390) roll requested
2024-11-17T20:36:21,817 INFO  [regionserver/0b0b0ae8de03:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0b0b0ae8de03%2C35963%2C1731875761298.1731875781817
2024-11-17T20:36:21,823 DEBUG [regionserver/0b0b0ae8de03:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/WALs/0b0b0ae8de03,35963,1731875761298/0b0b0ae8de03%2C35963%2C1731875761298.1731875776390 newFile=hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/WALs/0b0b0ae8de03,35963,1731875761298/0b0b0ae8de03%2C35963%2C1731875761298.1731875781817
2024-11-17T20:36:21,823 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:36:21,824 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:36:21,824 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:36:21,824 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:36:21,824 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:36:21,824 INFO  [regionserver/0b0b0ae8de03:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/WALs/0b0b0ae8de03,35963,1731875761298/0b0b0ae8de03%2C35963%2C1731875761298.1731875776390 with entries=2, filesize=2.37 KB; new WAL /user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/WALs/0b0b0ae8de03,35963,1731875761298/0b0b0ae8de03%2C35963%2C1731875761298.1731875781817
2024-11-17T20:36:21,824 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36439,DS-658f96ea-3d36-4123-8e44-dc53dc482242,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:36:21,824 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36439,DS-658f96ea-3d36-4123-8e44-dc53dc482242,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:36:21,825 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/WALs/0b0b0ae8de03,35963,1731875761298/0b0b0ae8de03%2C35963%2C1731875761298.1731875776390
2024-11-17T20:36:21,825 WARN  [IPC Server handler 4 on default port 46233 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/WALs/0b0b0ae8de03,35963,1731875761298/0b0b0ae8de03%2C35963%2C1731875761298.1731875776390 has not been closed. Lease recovery is in progress. RecoveryId = 1020 for block blk_1073741837_1018
2024-11-17T20:36:21,825 DEBUG [regionserver/0b0b0ae8de03:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43029:43029),(127.0.0.1/127.0.0.1:35499:35499)]
2024-11-17T20:36:21,825 DEBUG [regionserver/0b0b0ae8de03:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/WALs/0b0b0ae8de03,35963,1731875761298/0b0b0ae8de03%2C35963%2C1731875761298.1731875776390 is not closed yet, will try archiving it next time
2024-11-17T20:36:21,825 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/WALs/0b0b0ae8de03,35963,1731875761298/0b0b0ae8de03%2C35963%2C1731875761298.1731875776390 after 0ms
2024-11-17T20:36:21,948 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:22,775 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:22,791 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:22,949 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:23,776 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:23,792 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:23,827 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0b0b0ae8de03%2C35963%2C1731875761298.1731875783827
2024-11-17T20:36:23,834 DEBUG [Time-limited test {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/WALs/0b0b0ae8de03,35963,1731875761298/0b0b0ae8de03%2C35963%2C1731875761298.1731875781817 newFile=hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/WALs/0b0b0ae8de03,35963,1731875761298/0b0b0ae8de03%2C35963%2C1731875761298.1731875783827
2024-11-17T20:36:23,834 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:36:23,834 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:36:23,834 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:36:23,834 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:36:23,834 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:36:23,835 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/WALs/0b0b0ae8de03,35963,1731875761298/0b0b0ae8de03%2C35963%2C1731875761298.1731875781817 with entries=1, filesize=1.23 KB; new WAL /user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/WALs/0b0b0ae8de03,35963,1731875761298/0b0b0ae8de03%2C35963%2C1731875761298.1731875783827
2024-11-17T20:36:23,837 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34593 is added to blk_1073741838_1019 (size=1264)
2024-11-17T20:36:23,837 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38053 is added to blk_1073741838_1019 (size=1264)
2024-11-17T20:36:23,838 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/WALs/0b0b0ae8de03,35963,1731875761298/0b0b0ae8de03%2C35963%2C1731875761298.1731875776390 is not closed yet, will try archiving it next time
2024-11-17T20:36:23,845 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35499:35499),(127.0.0.1/127.0.0.1:43029:43029)]
2024-11-17T20:36:23,845 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/WALs/0b0b0ae8de03,35963,1731875761298/0b0b0ae8de03%2C35963%2C1731875761298.1731875776390 is not closed yet, will try archiving it next time
2024-11-17T20:36:23,846 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/WALs/0b0b0ae8de03,35963,1731875761298/0b0b0ae8de03%2C35963%2C1731875761298.1731875762332
2024-11-17T20:36:23,846 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/WALs/0b0b0ae8de03,35963,1731875761298/0b0b0ae8de03%2C35963%2C1731875761298.1731875762332
2024-11-17T20:36:23,846 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/WALs/0b0b0ae8de03,35963,1731875761298/0b0b0ae8de03%2C35963%2C1731875761298.1731875762332 after 0ms
2024-11-17T20:36:23,846 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/WALs/0b0b0ae8de03,35963,1731875761298/0b0b0ae8de03%2C35963%2C1731875761298.1731875762332
2024-11-17T20:36:23,857 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #3: [\x00/METAFAMILY:HBASE::REGION_EVENT::REGION_OPEN/1731875763309/Put/vlen=218/seqid=0]
2024-11-17T20:36:23,857 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #4: [row1002/info:/1731875773037/Put/vlen=1045/seqid=0]
2024-11-17T20:36:23,857 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/WALs/0b0b0ae8de03,35963,1731875761298/0b0b0ae8de03%2C35963%2C1731875761298.1731875762332
2024-11-17T20:36:23,857 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/WALs/0b0b0ae8de03,35963,1731875761298/0b0b0ae8de03%2C35963%2C1731875761298.1731875776390
2024-11-17T20:36:23,857 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/WALs/0b0b0ae8de03,35963,1731875761298/0b0b0ae8de03%2C35963%2C1731875761298.1731875776390
2024-11-17T20:36:23,858 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/WALs/0b0b0ae8de03,35963,1731875761298/0b0b0ae8de03%2C35963%2C1731875761298.1731875776390 after 1ms
2024-11-17T20:36:23,858 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/WALs/0b0b0ae8de03,35963,1731875761298/0b0b0ae8de03%2C35963%2C1731875761298.1731875776390
2024-11-17T20:36:23,862 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #5: [row1003/info:/1731875776389/Put/vlen=1045/seqid=0]
2024-11-17T20:36:23,862 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #6: [row1004/info:/1731875778403/Put/vlen=1045/seqid=0]
2024-11-17T20:36:23,862 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/WALs/0b0b0ae8de03,35963,1731875761298/0b0b0ae8de03%2C35963%2C1731875761298.1731875776390
2024-11-17T20:36:23,862 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/WALs/0b0b0ae8de03,35963,1731875761298/0b0b0ae8de03%2C35963%2C1731875761298.1731875781817
2024-11-17T20:36:23,862 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/WALs/0b0b0ae8de03,35963,1731875761298/0b0b0ae8de03%2C35963%2C1731875761298.1731875781817
2024-11-17T20:36:23,862 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/WALs/0b0b0ae8de03,35963,1731875761298/0b0b0ae8de03%2C35963%2C1731875761298.1731875781817 after 0ms
2024-11-17T20:36:23,862 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/WALs/0b0b0ae8de03,35963,1731875761298/0b0b0ae8de03%2C35963%2C1731875761298.1731875781817
2024-11-17T20:36:23,866 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #7: [row1005/info:/1731875781816/Put/vlen=1045/seqid=0]
2024-11-17T20:36:23,867 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/WALs/0b0b0ae8de03,35963,1731875761298/0b0b0ae8de03%2C35963%2C1731875761298.1731875783827
2024-11-17T20:36:23,867 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/WALs/0b0b0ae8de03,35963,1731875761298/0b0b0ae8de03%2C35963%2C1731875761298.1731875783827
2024-11-17T20:36:23,867 WARN  [IPC Server handler 3 on default port 46233 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/WALs/0b0b0ae8de03,35963,1731875761298/0b0b0ae8de03%2C35963%2C1731875761298.1731875783827 has not been closed. Lease recovery is in progress. RecoveryId = 1022 for block blk_1073741839_1021
2024-11-17T20:36:23,867 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/WALs/0b0b0ae8de03,35963,1731875761298/0b0b0ae8de03%2C35963%2C1731875761298.1731875783827 after 0ms
2024-11-17T20:36:23,950 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:24,126 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741837_1018: GenerationStamp not matched, existing replica is blk_1073741837_1016
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-11-17T20:36:24,353 WARN  [ResponseProcessor for block BP-1977883270-172.17.0.2-1731875758461:blk_1073741839_1021 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1977883270-172.17.0.2-1731875758461:blk_1073741839_1021
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:36:24,353 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1765916636_22 at /127.0.0.1:42450 [Receiving block BP-1977883270-172.17.0.2-1731875758461:blk_1073741839_1021] {}] datanode.DataXceiver(331): 127.0.0.1:34593:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:42450 dst: /127.0.0.1:34593
java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:34593 remote=/127.0.0.1:42450]. Total timeout mills is 60000, 59480 millis timeout left.
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:36:24,353 WARN  [DataStreamer for file /user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/WALs/0b0b0ae8de03,35963,1731875761298/0b0b0ae8de03%2C35963%2C1731875761298.1731875783827 block BP-1977883270-172.17.0.2-1731875758461:blk_1073741839_1021 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1977883270-172.17.0.2-1731875758461:blk_1073741839_1021 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34593,DS-5091058c-e4c0-4649-ad99-02c88df21742,DISK], DatanodeInfoWithStorage[127.0.0.1:38053,DS-658f96ea-3d36-4123-8e44-dc53dc482242,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:34593,DS-5091058c-e4c0-4649-ad99-02c88df21742,DISK]) is bad.
2024-11-17T20:36:24,353 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1765916636_22 at /127.0.0.1:35014 [Receiving block BP-1977883270-172.17.0.2-1731875758461:blk_1073741839_1021] {}] datanode.DataXceiver(331): 127.0.0.1:38053:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:35014 dst: /127.0.0.1:38053
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:36:24,355 WARN  [DataStreamer for file /user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/WALs/0b0b0ae8de03,35963,1731875761298/0b0b0ae8de03%2C35963%2C1731875761298.1731875783827 block BP-1977883270-172.17.0.2-1731875758461:blk_1073741839_1021 {}] hdfs.DataStreamer(859): DataStreamer Exception
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1977883270-172.17.0.2-1731875758461:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:36:24,359 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34593 is added to blk_1073741839_1022 (size=85)
2024-11-17T20:36:24,777 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:24,792 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:24,950 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:25,777 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:25,793 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:25,827 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/WALs/0b0b0ae8de03,35963,1731875761298/0b0b0ae8de03%2C35963%2C1731875761298.1731875776390 after 4002ms
2024-11-17T20:36:25,951 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:26,778 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:26,794 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:26,952 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:27,779 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:27,794 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:27,868 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/WALs/0b0b0ae8de03,35963,1731875761298/0b0b0ae8de03%2C35963%2C1731875761298.1731875783827 after 4001ms
2024-11-17T20:36:27,868 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/WALs/0b0b0ae8de03,35963,1731875761298/0b0b0ae8de03%2C35963%2C1731875761298.1731875783827
2024-11-17T20:36:27,873 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/WALs/0b0b0ae8de03,35963,1731875761298/0b0b0ae8de03%2C35963%2C1731875761298.1731875783827
2024-11-17T20:36:27,873 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.74 KB heapSize=3.77 KB
2024-11-17T20:36:27,873 ERROR [FSHLog-0-hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6-prefix:0b0b0ae8de03,35963,1731875761298.meta {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45599,DS-5091058c-e4c0-4649-ad99-02c88df21742,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:36:27,874 WARN  [FSHLog-0-hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6-prefix:0b0b0ae8de03,35963,1731875761298.meta {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45599,DS-5091058c-e4c0-4649-ad99-02c88df21742,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:36:27,874 DEBUG [regionserver/0b0b0ae8de03:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 0b0b0ae8de03%2C35963%2C1731875761298.meta:.meta(num 1731875762780) roll requested
2024-11-17T20:36:27,874 INFO  [regionserver/0b0b0ae8de03:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0b0b0ae8de03%2C35963%2C1731875761298.meta.1731875787874.meta
2024-11-17T20:36:27,880 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:36:27,880 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:36:27,880 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:36:27,880 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:36:27,880 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:36:27,881 INFO  [regionserver/0b0b0ae8de03:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/WALs/0b0b0ae8de03,35963,1731875761298/0b0b0ae8de03%2C35963%2C1731875761298.meta.1731875762780.meta with entries=8, filesize=2.36 KB; new WAL /user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/WALs/0b0b0ae8de03,35963,1731875761298/0b0b0ae8de03%2C35963%2C1731875761298.meta.1731875787874.meta
2024-11-17T20:36:27,881 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45599,DS-5091058c-e4c0-4649-ad99-02c88df21742,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:36:27,881 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45599,DS-5091058c-e4c0-4649-ad99-02c88df21742,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:36:27,881 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/WALs/0b0b0ae8de03,35963,1731875761298/0b0b0ae8de03%2C35963%2C1731875761298.meta.1731875762780.meta
2024-11-17T20:36:27,882 DEBUG [regionserver/0b0b0ae8de03:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43029:43029),(127.0.0.1/127.0.0.1:35499:35499)]
2024-11-17T20:36:27,882 WARN  [IPC Server handler 4 on default port 46233 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/WALs/0b0b0ae8de03,35963,1731875761298/0b0b0ae8de03%2C35963%2C1731875761298.meta.1731875762780.meta has not been closed. Lease recovery is in progress. RecoveryId = 1024 for block blk_1073741834_1013
2024-11-17T20:36:27,882 DEBUG [regionserver/0b0b0ae8de03:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/WALs/0b0b0ae8de03,35963,1731875761298/0b0b0ae8de03%2C35963%2C1731875761298.meta.1731875762780.meta is not closed yet, will try archiving it next time
2024-11-17T20:36:27,882 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/WALs/0b0b0ae8de03,35963,1731875761298/0b0b0ae8de03%2C35963%2C1731875761298.meta.1731875762780.meta after 1ms
2024-11-17T20:36:27,901 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/data/hbase/meta/1588230740/.tmp/info/01820b3f1dd04cb08f4027e7c0d93518 is 207, key is TestLogRolling-testLogRollOnPipelineRestart,,1731875762945.2e44545a7215dcd0d833f62680b5cba5./info:regioninfo/1731875763313/Put/seqid=0
2024-11-17T20:36:27,910 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38053 is added to blk_1073741841_1025 (size=7125)
2024-11-17T20:36:27,910 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34593 is added to blk_1073741841_1025 (size=7125)
2024-11-17T20:36:27,910 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.52 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/data/hbase/meta/1588230740/.tmp/info/01820b3f1dd04cb08f4027e7c0d93518
2024-11-17T20:36:27,930 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/data/hbase/meta/1588230740/.tmp/ns/cd0acd2cc8664237bf8d5af690a0baed is 43, key is default/ns:d/1731875762859/Put/seqid=0
2024-11-17T20:36:27,934 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34593 is added to blk_1073741842_1026 (size=5153)
2024-11-17T20:36:27,934 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38053 is added to blk_1073741842_1026 (size=5153)
2024-11-17T20:36:27,935 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/data/hbase/meta/1588230740/.tmp/ns/cd0acd2cc8664237bf8d5af690a0baed
2024-11-17T20:36:27,952 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:27,957 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/data/hbase/meta/1588230740/.tmp/table/adf5e4b17ff147b08b1c9f7194f6054a is 79, key is TestLogRolling-testLogRollOnPipelineRestart/table:state/1731875763323/Put/seqid=0
2024-11-17T20:36:27,961 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38053 is added to blk_1073741843_1027 (size=5438)
2024-11-17T20:36:27,961 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34593 is added to blk_1073741843_1027 (size=5438)
2024-11-17T20:36:27,962 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=150 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/data/hbase/meta/1588230740/.tmp/table/adf5e4b17ff147b08b1c9f7194f6054a
2024-11-17T20:36:27,968 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/data/hbase/meta/1588230740/.tmp/info/01820b3f1dd04cb08f4027e7c0d93518 as hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/data/hbase/meta/1588230740/info/01820b3f1dd04cb08f4027e7c0d93518
2024-11-17T20:36:27,974 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/data/hbase/meta/1588230740/info/01820b3f1dd04cb08f4027e7c0d93518, entries=10, sequenceid=11, filesize=7.0 K
2024-11-17T20:36:27,975 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/data/hbase/meta/1588230740/.tmp/ns/cd0acd2cc8664237bf8d5af690a0baed as hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/data/hbase/meta/1588230740/ns/cd0acd2cc8664237bf8d5af690a0baed
2024-11-17T20:36:27,981 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/data/hbase/meta/1588230740/ns/cd0acd2cc8664237bf8d5af690a0baed, entries=2, sequenceid=11, filesize=5.0 K
2024-11-17T20:36:27,983 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/data/hbase/meta/1588230740/.tmp/table/adf5e4b17ff147b08b1c9f7194f6054a as hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/data/hbase/meta/1588230740/table/adf5e4b17ff147b08b1c9f7194f6054a
2024-11-17T20:36:27,990 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/data/hbase/meta/1588230740/table/adf5e4b17ff147b08b1c9f7194f6054a, entries=2, sequenceid=11, filesize=5.3 K
2024-11-17T20:36:27,991 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.74 KB/1782, heapSize ~3.48 KB/3560, currentSize=0 B/0 for 1588230740 in 118ms, sequenceid=11, compaction requested=false
2024-11-17T20:36:27,992 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1588230740:

2024-11-17T20:36:27,992 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing 2e44545a7215dcd0d833f62680b5cba5 1/1 column families, dataSize=4.20 KB heapSize=4.75 KB
2024-11-17T20:36:27,992 ERROR [FSHLog-0-hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6-prefix:0b0b0ae8de03,35963,1731875761298 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1977883270-172.17.0.2-1731875758461:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:36:27,993 WARN  [FSHLog-0-hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6-prefix:0b0b0ae8de03,35963,1731875761298 {}] wal.AbstractFSWAL(2174): append entry failed
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1977883270-172.17.0.2-1731875758461:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:36:27,993 DEBUG [regionserver/0b0b0ae8de03:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 0b0b0ae8de03%2C35963%2C1731875761298:(num 1731875783827) roll requested
2024-11-17T20:36:27,993 INFO  [regionserver/0b0b0ae8de03:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0b0b0ae8de03%2C35963%2C1731875761298.1731875787993
2024-11-17T20:36:27,998 DEBUG [regionserver/0b0b0ae8de03:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/WALs/0b0b0ae8de03,35963,1731875761298/0b0b0ae8de03%2C35963%2C1731875761298.1731875783827 newFile=hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/WALs/0b0b0ae8de03,35963,1731875761298/0b0b0ae8de03%2C35963%2C1731875761298.1731875787993
2024-11-17T20:36:27,999 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:36:27,999 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:36:27,999 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:36:27,999 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:36:27,999 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:36:27,999 INFO  [regionserver/0b0b0ae8de03:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/WALs/0b0b0ae8de03,35963,1731875761298/0b0b0ae8de03%2C35963%2C1731875761298.1731875783827 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/WALs/0b0b0ae8de03,35963,1731875761298/0b0b0ae8de03%2C35963%2C1731875761298.1731875787993
2024-11-17T20:36:27,999 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1977883270-172.17.0.2-1731875758461:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:36:28,000 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1977883270-172.17.0.2-1731875758461:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:36:28,000 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/WALs/0b0b0ae8de03,35963,1731875761298/0b0b0ae8de03%2C35963%2C1731875761298.1731875783827
2024-11-17T20:36:28,000 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/WALs/0b0b0ae8de03,35963,1731875761298/0b0b0ae8de03%2C35963%2C1731875761298.1731875783827 after 0ms
2024-11-17T20:36:28,006 DEBUG [regionserver/0b0b0ae8de03:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35499:35499),(127.0.0.1/127.0.0.1:43029:43029)]
2024-11-17T20:36:28,006 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/WALs/0b0b0ae8de03,35963,1731875761298/0b0b0ae8de03%2C35963%2C1731875761298.1731875783827 to hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/oldWALs/0b0b0ae8de03%2C35963%2C1731875761298.1731875783827
2024-11-17T20:36:28,023 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/data/default/TestLogRolling-testLogRollOnPipelineRestart/2e44545a7215dcd0d833f62680b5cba5/.tmp/info/b9500f5b52ca4934a04c4eae34090386 is 1080, key is row1002/info:/1731875773037/Put/seqid=0
2024-11-17T20:36:28,028 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34593 is added to blk_1073741845_1029 (size=9270)
2024-11-17T20:36:28,028 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38053 is added to blk_1073741845_1029 (size=9270)
2024-11-17T20:36:28,029 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.20 KB at sequenceid=8 (bloomFilter=true), to=hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/data/default/TestLogRolling-testLogRollOnPipelineRestart/2e44545a7215dcd0d833f62680b5cba5/.tmp/info/b9500f5b52ca4934a04c4eae34090386
2024-11-17T20:36:28,035 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/data/default/TestLogRolling-testLogRollOnPipelineRestart/2e44545a7215dcd0d833f62680b5cba5/.tmp/info/b9500f5b52ca4934a04c4eae34090386 as hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/data/default/TestLogRolling-testLogRollOnPipelineRestart/2e44545a7215dcd0d833f62680b5cba5/info/b9500f5b52ca4934a04c4eae34090386
2024-11-17T20:36:28,042 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/data/default/TestLogRolling-testLogRollOnPipelineRestart/2e44545a7215dcd0d833f62680b5cba5/info/b9500f5b52ca4934a04c4eae34090386, entries=4, sequenceid=8, filesize=9.1 K
2024-11-17T20:36:28,043 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~4.20 KB/4304, heapSize ~4.73 KB/4848, currentSize=0 B/0 for 2e44545a7215dcd0d833f62680b5cba5 in 51ms, sequenceid=8, compaction requested=false
2024-11-17T20:36:28,043 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 2e44545a7215dcd0d833f62680b5cba5:

2024-11-17T20:36:28,048 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-17T20:36:28,048 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-17T20:36:28,049 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-17T20:36:28,049 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T20:36:28,049 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T20:36:28,049 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-17T20:36:28,049 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-17T20:36:28,049 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=516382525, stopped=false
2024-11-17T20:36:28,049 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=0b0b0ae8de03,36429,1731875761121
2024-11-17T20:36:28,119 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35963-0x1014a82e7730001, quorum=127.0.0.1:49481, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-17T20:36:28,119 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35963-0x1014a82e7730001, quorum=127.0.0.1:49481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:36:28,119 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36429-0x1014a82e7730000, quorum=127.0.0.1:49481, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-17T20:36:28,119 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-17T20:36:28,119 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36429-0x1014a82e7730000, quorum=127.0.0.1:49481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:36:28,119 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36429-0x1014a82e7730000, quorum=127.0.0.1:49481, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-17T20:36:28,119 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35963-0x1014a82e7730001, quorum=127.0.0.1:49481, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-17T20:36:28,119 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-17T20:36:28,120 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-17T20:36:28,120 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T20:36:28,120 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '0b0b0ae8de03,35963,1731875761298' *****
2024-11-17T20:36:28,120 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-17T20:36:28,120 INFO  [RS:0;0b0b0ae8de03:35963 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-17T20:36:28,120 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-17T20:36:28,120 INFO  [RS:0;0b0b0ae8de03:35963 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-17T20:36:28,120 INFO  [RS:0;0b0b0ae8de03:35963 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-17T20:36:28,120 INFO  [RS:0;0b0b0ae8de03:35963 {}] regionserver.HRegionServer(3091): Received CLOSE for 2e44545a7215dcd0d833f62680b5cba5
2024-11-17T20:36:28,121 INFO  [RS:0;0b0b0ae8de03:35963 {}] regionserver.HRegionServer(959): stopping server 0b0b0ae8de03,35963,1731875761298
2024-11-17T20:36:28,121 INFO  [RS:0;0b0b0ae8de03:35963 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-17T20:36:28,121 INFO  [RS:0;0b0b0ae8de03:35963 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;0b0b0ae8de03:35963.
2024-11-17T20:36:28,121 DEBUG [RS:0;0b0b0ae8de03:35963 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-17T20:36:28,121 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 2e44545a7215dcd0d833f62680b5cba5, disabling compactions & flushes
2024-11-17T20:36:28,121 DEBUG [RS:0;0b0b0ae8de03:35963 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T20:36:28,121 INFO  [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1731875762945.2e44545a7215dcd0d833f62680b5cba5.
2024-11-17T20:36:28,121 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1731875762945.2e44545a7215dcd0d833f62680b5cba5.
2024-11-17T20:36:28,121 INFO  [RS:0;0b0b0ae8de03:35963 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-17T20:36:28,121 INFO  [RS:0;0b0b0ae8de03:35963 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-17T20:36:28,121 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1731875762945.2e44545a7215dcd0d833f62680b5cba5. after waiting 0 ms
2024-11-17T20:36:28,121 INFO  [RS:0;0b0b0ae8de03:35963 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-17T20:36:28,121 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1731875762945.2e44545a7215dcd0d833f62680b5cba5.
2024-11-17T20:36:28,121 INFO  [RS:0;0b0b0ae8de03:35963 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-17T20:36:28,121 INFO  [RS:0;0b0b0ae8de03:35963 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-11-17T20:36:28,121 DEBUG [RS:0;0b0b0ae8de03:35963 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 2e44545a7215dcd0d833f62680b5cba5=TestLogRolling-testLogRollOnPipelineRestart,,1731875762945.2e44545a7215dcd0d833f62680b5cba5.}
2024-11-17T20:36:28,121 DEBUG [RS:0;0b0b0ae8de03:35963 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 2e44545a7215dcd0d833f62680b5cba5
2024-11-17T20:36:28,121 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-17T20:36:28,121 INFO  [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-17T20:36:28,121 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-17T20:36:28,122 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-17T20:36:28,122 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-17T20:36:28,130 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/data/default/TestLogRolling-testLogRollOnPipelineRestart/2e44545a7215dcd0d833f62680b5cba5/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1
2024-11-17T20:36:28,131 INFO  [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnPipelineRestart,,1731875762945.2e44545a7215dcd0d833f62680b5cba5.
2024-11-17T20:36:28,131 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 2e44545a7215dcd0d833f62680b5cba5:
Waiting for close lock at 1731875788121Running coprocessor pre-close hooks at 1731875788121Disabling compacts and flushes for region at 1731875788121Disabling writes for close at 1731875788121Writing region close event to WAL at 1731875788122 (+1 ms)Running coprocessor post-close hooks at 1731875788131 (+9 ms)Closed at 1731875788131
2024-11-17T20:36:28,131 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRollOnPipelineRestart,,1731875762945.2e44545a7215dcd0d833f62680b5cba5.
2024-11-17T20:36:28,134 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-11-17T20:36:28,134 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-17T20:36:28,135 INFO  [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-17T20:36:28,135 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731875788121Running coprocessor pre-close hooks at 1731875788121Disabling compacts and flushes for region at 1731875788121Disabling writes for close at 1731875788122 (+1 ms)Writing region close event to WAL at 1731875788130 (+8 ms)Running coprocessor post-close hooks at 1731875788134 (+4 ms)Closed at 1731875788135 (+1 ms)
2024-11-17T20:36:28,135 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-17T20:36:28,194 INFO  [regionserver/0b0b0ae8de03:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-11-17T20:36:28,194 INFO  [regionserver/0b0b0ae8de03:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-11-17T20:36:28,196 INFO  [regionserver/0b0b0ae8de03:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-17T20:36:28,321 INFO  [RS:0;0b0b0ae8de03:35963 {}] regionserver.HRegionServer(976): stopping server 0b0b0ae8de03,35963,1731875761298; all regions closed.
2024-11-17T20:36:28,322 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:36:28,322 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:36:28,322 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:36:28,322 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:36:28,322 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:36:28,325 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34593 is added to blk_1073741840_1023 (size=825)
2024-11-17T20:36:28,325 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38053 is added to blk_1073741840_1023 (size=825)
2024-11-17T20:36:28,779 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:28,795 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:28,953 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:29,348 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741834_1013: GenerationStamp not matched, existing replica is blk_1073741834_1010
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-11-17T20:36:29,780 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:29,796 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:29,954 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:30,781 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:30,797 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:30,954 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:31,093 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-17T20:36:31,782 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:31,797 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:31,884 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/WALs/0b0b0ae8de03,35963,1731875761298/0b0b0ae8de03%2C35963%2C1731875761298.meta.1731875762780.meta after 4003ms
2024-11-17T20:36:31,885 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/WALs/0b0b0ae8de03,35963,1731875761298/0b0b0ae8de03%2C35963%2C1731875761298.meta.1731875762780.meta to hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/oldWALs/0b0b0ae8de03%2C35963%2C1731875761298.meta.1731875762780.meta
2024-11-17T20:36:31,893 DEBUG [RS:0;0b0b0ae8de03:35963 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/oldWALs
2024-11-17T20:36:31,893 INFO  [RS:0;0b0b0ae8de03:35963 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 0b0b0ae8de03%2C35963%2C1731875761298.meta:.meta(num 1731875787874)
2024-11-17T20:36:31,893 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:36:31,893 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:36:31,893 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:36:31,894 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:36:31,894 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:36:31,896 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38053 is added to blk_1073741844_1028 (size=1162)
2024-11-17T20:36:31,896 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34593 is added to blk_1073741844_1028 (size=1162)
2024-11-17T20:36:31,901 DEBUG [RS:0;0b0b0ae8de03:35963 {}] wal.AbstractFSWAL(1256): Moved 4 WAL file(s) to /user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/oldWALs
2024-11-17T20:36:31,901 INFO  [RS:0;0b0b0ae8de03:35963 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 0b0b0ae8de03%2C35963%2C1731875761298:(num 1731875787993)
2024-11-17T20:36:31,901 DEBUG [RS:0;0b0b0ae8de03:35963 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T20:36:31,901 INFO  [RS:0;0b0b0ae8de03:35963 {}] regionserver.LeaseManager(133): Closed leases
2024-11-17T20:36:31,901 INFO  [RS:0;0b0b0ae8de03:35963 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-17T20:36:31,902 INFO  [RS:0;0b0b0ae8de03:35963 {}] hbase.ChoreService(370): Chore service for: regionserver/0b0b0ae8de03:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-17T20:36:31,902 INFO  [RS:0;0b0b0ae8de03:35963 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-17T20:36:31,902 INFO  [regionserver/0b0b0ae8de03:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-17T20:36:31,902 INFO  [RS:0;0b0b0ae8de03:35963 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35963
2024-11-17T20:36:31,921 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35963-0x1014a82e7730001, quorum=127.0.0.1:49481, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/0b0b0ae8de03,35963,1731875761298
2024-11-17T20:36:31,921 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36429-0x1014a82e7730000, quorum=127.0.0.1:49481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-17T20:36:31,921 INFO  [RS:0;0b0b0ae8de03:35963 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-17T20:36:31,932 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [0b0b0ae8de03,35963,1731875761298]
2024-11-17T20:36:31,942 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/0b0b0ae8de03,35963,1731875761298 already deleted, retry=false
2024-11-17T20:36:31,942 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 0b0b0ae8de03,35963,1731875761298 expired; onlineServers=0
2024-11-17T20:36:31,942 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '0b0b0ae8de03,36429,1731875761121' *****
2024-11-17T20:36:31,942 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-17T20:36:31,943 INFO  [M:0;0b0b0ae8de03:36429 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-17T20:36:31,943 INFO  [M:0;0b0b0ae8de03:36429 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-17T20:36:31,943 DEBUG [M:0;0b0b0ae8de03:36429 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-17T20:36:31,943 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-17T20:36:31,943 DEBUG [M:0;0b0b0ae8de03:36429 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-17T20:36:31,943 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster-HFileCleaner.large.0-1731875762116 {}] cleaner.HFileCleaner(306): Exit Thread[master/0b0b0ae8de03:0:becomeActiveMaster-HFileCleaner.large.0-1731875762116,5,FailOnTimeoutGroup]
2024-11-17T20:36:31,943 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster-HFileCleaner.small.0-1731875762117 {}] cleaner.HFileCleaner(306): Exit Thread[master/0b0b0ae8de03:0:becomeActiveMaster-HFileCleaner.small.0-1731875762117,5,FailOnTimeoutGroup]
2024-11-17T20:36:31,943 INFO  [M:0;0b0b0ae8de03:36429 {}] hbase.ChoreService(370): Chore service for: master/0b0b0ae8de03:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-17T20:36:31,944 INFO  [M:0;0b0b0ae8de03:36429 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-17T20:36:31,944 DEBUG [M:0;0b0b0ae8de03:36429 {}] master.HMaster(1795): Stopping service threads
2024-11-17T20:36:31,944 INFO  [M:0;0b0b0ae8de03:36429 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-17T20:36:31,944 INFO  [M:0;0b0b0ae8de03:36429 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-17T20:36:31,944 INFO  [M:0;0b0b0ae8de03:36429 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-17T20:36:31,944 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-17T20:36:31,953 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36429-0x1014a82e7730000, quorum=127.0.0.1:49481, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-17T20:36:31,953 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36429-0x1014a82e7730000, quorum=127.0.0.1:49481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:36:31,953 DEBUG [M:0;0b0b0ae8de03:36429 {}] zookeeper.ZKUtil(347): master:36429-0x1014a82e7730000, quorum=127.0.0.1:49481, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-17T20:36:31,953 WARN  [M:0;0b0b0ae8de03:36429 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-17T20:36:31,954 INFO  [M:0;0b0b0ae8de03:36429 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/.lastflushedseqids
2024-11-17T20:36:31,955 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:31,965 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38053 is added to blk_1073741846_1030 (size=120)
2024-11-17T20:36:31,965 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34593 is added to blk_1073741846_1030 (size=120)
2024-11-17T20:36:31,965 INFO  [M:0;0b0b0ae8de03:36429 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-17T20:36:31,966 INFO  [M:0;0b0b0ae8de03:36429 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-17T20:36:31,966 DEBUG [M:0;0b0b0ae8de03:36429 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-17T20:36:31,966 INFO  [M:0;0b0b0ae8de03:36429 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T20:36:31,966 DEBUG [M:0;0b0b0ae8de03:36429 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T20:36:31,966 DEBUG [M:0;0b0b0ae8de03:36429 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-17T20:36:31,966 DEBUG [M:0;0b0b0ae8de03:36429 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T20:36:31,966 INFO  [M:0;0b0b0ae8de03:36429 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.17 KB heapSize=29.16 KB
2024-11-17T20:36:31,967 ERROR [FSHLog-0-hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/MasterData-prefix:0b0b0ae8de03,36429,1731875761121 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45599,DS-5091058c-e4c0-4649-ad99-02c88df21742,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:36:31,967 WARN  [FSHLog-0-hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/MasterData-prefix:0b0b0ae8de03,36429,1731875761121 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45599,DS-5091058c-e4c0-4649-ad99-02c88df21742,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:36:31,967 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(198): WAL FSHLog 0b0b0ae8de03%2C36429%2C1731875761121:(num 1731875761855) roll requested
2024-11-17T20:36:31,967 INFO  [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0b0b0ae8de03%2C36429%2C1731875761121.1731875791967
2024-11-17T20:36:31,975 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:36:31,975 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:36:31,975 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:36:31,976 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:36:31,976 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:36:31,976 INFO  [master:store-WAL-Roller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/MasterData/WALs/0b0b0ae8de03,36429,1731875761121/0b0b0ae8de03%2C36429%2C1731875761121.1731875761855 with entries=53, filesize=26.62 KB; new WAL /user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/MasterData/WALs/0b0b0ae8de03,36429,1731875761121/0b0b0ae8de03%2C36429%2C1731875761121.1731875791967
2024-11-17T20:36:31,976 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45599,DS-5091058c-e4c0-4649-ad99-02c88df21742,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:36:31,977 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45599,DS-5091058c-e4c0-4649-ad99-02c88df21742,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T20:36:31,977 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/MasterData/WALs/0b0b0ae8de03,36429,1731875761121/0b0b0ae8de03%2C36429%2C1731875761121.1731875761855
2024-11-17T20:36:31,977 WARN  [IPC Server handler 3 on default port 46233 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/MasterData/WALs/0b0b0ae8de03,36429,1731875761121/0b0b0ae8de03%2C36429%2C1731875761121.1731875761855 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741830_1015
2024-11-17T20:36:31,977 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/MasterData/WALs/0b0b0ae8de03,36429,1731875761121/0b0b0ae8de03%2C36429%2C1731875761121.1731875761855 after 0ms
2024-11-17T20:36:31,978 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43029:43029),(127.0.0.1/127.0.0.1:35499:35499)]
2024-11-17T20:36:31,978 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(879): hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/MasterData/WALs/0b0b0ae8de03,36429,1731875761121/0b0b0ae8de03%2C36429%2C1731875761121.1731875761855 is not closed yet, will try archiving it next time
2024-11-17T20:36:32,002 DEBUG [M:0;0b0b0ae8de03:36429 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/74a78370fd684dacaf14853125058e09 is 82, key is hbase:meta,,1/info:regioninfo/1731875762805/Put/seqid=0
2024-11-17T20:36:32,006 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34593 is added to blk_1073741848_1033 (size=5672)
2024-11-17T20:36:32,007 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38053 is added to blk_1073741848_1033 (size=5672)
2024-11-17T20:36:32,007 INFO  [M:0;0b0b0ae8de03:36429 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/74a78370fd684dacaf14853125058e09
2024-11-17T20:36:32,025 DEBUG [M:0;0b0b0ae8de03:36429 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/716070a62e3549a38770a01ae352ec49 is 778, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731875763328/Put/seqid=0
2024-11-17T20:36:32,029 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34593 is added to blk_1073741849_1034 (size=6118)
2024-11-17T20:36:32,030 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38053 is added to blk_1073741849_1034 (size=6118)
2024-11-17T20:36:32,030 INFO  [M:0;0b0b0ae8de03:36429 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.57 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/716070a62e3549a38770a01ae352ec49
2024-11-17T20:36:32,032 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35963-0x1014a82e7730001, quorum=127.0.0.1:49481, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-17T20:36:32,032 INFO  [RS:0;0b0b0ae8de03:35963 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-17T20:36:32,032 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35963-0x1014a82e7730001, quorum=127.0.0.1:49481, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-17T20:36:32,032 INFO  [RS:0;0b0b0ae8de03:35963 {}] regionserver.HRegionServer(1031): Exiting; stopping=0b0b0ae8de03,35963,1731875761298; zookeeper connection closed.
2024-11-17T20:36:32,032 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@722f56cd {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@722f56cd
2024-11-17T20:36:32,032 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-17T20:36:32,048 DEBUG [M:0;0b0b0ae8de03:36429 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/33756c2985994550bc361eb5bc89f68b is 69, key is 0b0b0ae8de03,35963,1731875761298/rs:state/1731875762161/Put/seqid=0
2024-11-17T20:36:32,053 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38053 is added to blk_1073741850_1035 (size=5156)
2024-11-17T20:36:32,053 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34593 is added to blk_1073741850_1035 (size=5156)
2024-11-17T20:36:32,053 INFO  [M:0;0b0b0ae8de03:36429 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/33756c2985994550bc361eb5bc89f68b
2024-11-17T20:36:32,071 DEBUG [M:0;0b0b0ae8de03:36429 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/b76444cc37cb45f0aa46c639cf84a117 is 52, key is load_balancer_on/state:d/1731875762940/Put/seqid=0
2024-11-17T20:36:32,075 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38053 is added to blk_1073741851_1036 (size=5056)
2024-11-17T20:36:32,075 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34593 is added to blk_1073741851_1036 (size=5056)
2024-11-17T20:36:32,076 INFO  [M:0;0b0b0ae8de03:36429 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/b76444cc37cb45f0aa46c639cf84a117
2024-11-17T20:36:32,082 DEBUG [M:0;0b0b0ae8de03:36429 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/74a78370fd684dacaf14853125058e09 as hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/74a78370fd684dacaf14853125058e09
2024-11-17T20:36:32,087 INFO  [M:0;0b0b0ae8de03:36429 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/74a78370fd684dacaf14853125058e09, entries=8, sequenceid=56, filesize=5.5 K
2024-11-17T20:36:32,088 DEBUG [M:0;0b0b0ae8de03:36429 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/716070a62e3549a38770a01ae352ec49 as hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/716070a62e3549a38770a01ae352ec49
2024-11-17T20:36:32,093 INFO  [M:0;0b0b0ae8de03:36429 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/716070a62e3549a38770a01ae352ec49, entries=6, sequenceid=56, filesize=6.0 K
2024-11-17T20:36:32,094 DEBUG [M:0;0b0b0ae8de03:36429 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/33756c2985994550bc361eb5bc89f68b as hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/33756c2985994550bc361eb5bc89f68b
2024-11-17T20:36:32,099 INFO  [M:0;0b0b0ae8de03:36429 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/33756c2985994550bc361eb5bc89f68b, entries=1, sequenceid=56, filesize=5.0 K
2024-11-17T20:36:32,100 DEBUG [M:0;0b0b0ae8de03:36429 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/b76444cc37cb45f0aa46c639cf84a117 as hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/b76444cc37cb45f0aa46c639cf84a117
2024-11-17T20:36:32,106 INFO  [M:0;0b0b0ae8de03:36429 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/b76444cc37cb45f0aa46c639cf84a117, entries=1, sequenceid=56, filesize=4.9 K
2024-11-17T20:36:32,107 INFO  [M:0;0b0b0ae8de03:36429 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.17 KB/23726, heapSize ~29.10 KB/29800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 141ms, sequenceid=56, compaction requested=false
2024-11-17T20:36:32,108 INFO  [M:0;0b0b0ae8de03:36429 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T20:36:32,109 DEBUG [M:0;0b0b0ae8de03:36429 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731875791966Disabling compacts and flushes for region at 1731875791966Disabling writes for close at 1731875791966Obtaining lock to block concurrent updates at 1731875791966Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731875791966Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23726, getHeapSize=29800, getOffHeapSize=0, getCellsCount=67 at 1731875791967 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731875791978 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731875791979 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731875792001 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731875792002 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731875792012 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731875792025 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731875792025Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731875792034 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731875792048 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731875792048Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731875792058 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731875792070 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731875792070Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5b1ee996: reopening flushed file at 1731875792081 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@c0bb2ed: reopening flushed file at 1731875792087 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7840a6a7: reopening flushed file at 1731875792093 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@476d6740: reopening flushed file at 1731875792099 (+6 ms)Finished flush of dataSize ~23.17 KB/23726, heapSize ~29.10 KB/29800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 141ms, sequenceid=56, compaction requested=false at 1731875792107 (+8 ms)Writing region close event to WAL at 1731875792108 (+1 ms)Closed at 1731875792108
2024-11-17T20:36:32,109 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:36:32,109 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:36:32,109 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:36:32,109 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:36:32,109 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:36:32,111 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34593 is added to blk_1073741847_1031 (size=757)
2024-11-17T20:36:32,112 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38053 is added to blk_1073741847_1031 (size=757)
2024-11-17T20:36:32,348 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741830_1015: GenerationStamp not matched, existing replica is blk_1073741830_1006
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-11-17T20:36:32,783 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:32,798 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:32,956 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:33,131 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:36:33,131 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:36:33,154 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:36:33,154 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:36:33,154 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:36:33,154 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:36:33,154 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:36:33,155 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:36:33,158 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:36:33,158 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:36:33,158 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:36:33,160 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:36:33,164 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:36:33,164 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:36:33,667 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-17T20:36:33,668 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:36:33,669 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:36:33,669 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:36:33,670 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:36:33,699 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:36:33,699 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:36:33,700 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:36:33,700 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:36:33,700 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:36:33,701 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:36:33,707 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:36:33,708 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:36:33,708 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:36:33,711 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:36:33,783 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:33,799 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:33,803 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-17T20:36:33,803 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-17T20:36:33,803 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-11-17T20:36:33,803 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart
2024-11-17T20:36:33,957 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:34,784 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:34,799 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:34,957 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:35,785 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:35,800 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:35,958 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:35,978 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/MasterData/WALs/0b0b0ae8de03,36429,1731875761121/0b0b0ae8de03%2C36429%2C1731875761121.1731875761855 after 4001ms
2024-11-17T20:36:35,979 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/MasterData/WALs/0b0b0ae8de03,36429,1731875761121/0b0b0ae8de03%2C36429%2C1731875761121.1731875761855 to hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/MasterData/oldWALs/0b0b0ae8de03%2C36429%2C1731875761121.1731875761855
2024-11-17T20:36:35,982 INFO  [WAL-Archive-0 {}] region.MasterRegionUtils(50): Moved hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/MasterData/oldWALs/0b0b0ae8de03%2C36429%2C1731875761121.1731875761855 to hdfs://localhost:46233/user/jenkins/test-data/7005b2da-f76b-8ec7-ca2f-26119dfefaa6/oldWALs/0b0b0ae8de03%2C36429%2C1731875761121.1731875761855$masterlocalwal$
2024-11-17T20:36:35,982 INFO  [M:0;0b0b0ae8de03:36429 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-17T20:36:35,982 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-17T20:36:35,982 INFO  [M:0;0b0b0ae8de03:36429 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36429
2024-11-17T20:36:35,982 INFO  [M:0;0b0b0ae8de03:36429 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-17T20:36:36,134 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36429-0x1014a82e7730000, quorum=127.0.0.1:49481, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-17T20:36:36,134 INFO  [M:0;0b0b0ae8de03:36429 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-17T20:36:36,134 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36429-0x1014a82e7730000, quorum=127.0.0.1:49481, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-17T20:36:36,138 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@39e17fd3{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T20:36:36,138 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5b16a6e2{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-17T20:36:36,138 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-17T20:36:36,139 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7fc558ce{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-17T20:36:36,139 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@160f7ba9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d2981510-59e6-ff69-efd1-5906a8a92785/hadoop.log.dir/,STOPPED}
2024-11-17T20:36:36,140 WARN  [BP-1977883270-172.17.0.2-1731875758461 heartbeating to localhost/127.0.0.1:46233 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-17T20:36:36,140 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-17T20:36:36,140 WARN  [BP-1977883270-172.17.0.2-1731875758461 heartbeating to localhost/127.0.0.1:46233 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1977883270-172.17.0.2-1731875758461 (Datanode Uuid 35b41824-7467-4b8c-81a8-a7878b27f81a) service to localhost/127.0.0.1:46233
2024-11-17T20:36:36,140 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-17T20:36:36,141 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d2981510-59e6-ff69-efd1-5906a8a92785/cluster_a0b2892d-6b1d-843d-7c12-ddaf29516f86/data/data3/current/BP-1977883270-172.17.0.2-1731875758461 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T20:36:36,142 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d2981510-59e6-ff69-efd1-5906a8a92785/cluster_a0b2892d-6b1d-843d-7c12-ddaf29516f86/data/data4/current/BP-1977883270-172.17.0.2-1731875758461 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T20:36:36,142 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-17T20:36:36,144 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@48919a97{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T20:36:36,145 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4778d192{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-17T20:36:36,145 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-17T20:36:36,145 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6d237fac{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-17T20:36:36,145 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4a79eeb5{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d2981510-59e6-ff69-efd1-5906a8a92785/hadoop.log.dir/,STOPPED}
2024-11-17T20:36:36,146 WARN  [BP-1977883270-172.17.0.2-1731875758461 heartbeating to localhost/127.0.0.1:46233 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-17T20:36:36,146 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-17T20:36:36,146 WARN  [BP-1977883270-172.17.0.2-1731875758461 heartbeating to localhost/127.0.0.1:46233 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1977883270-172.17.0.2-1731875758461 (Datanode Uuid ba381e1e-86b5-491b-b28d-cf1370bb3873) service to localhost/127.0.0.1:46233
2024-11-17T20:36:36,146 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-17T20:36:36,147 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d2981510-59e6-ff69-efd1-5906a8a92785/cluster_a0b2892d-6b1d-843d-7c12-ddaf29516f86/data/data1/current/BP-1977883270-172.17.0.2-1731875758461 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T20:36:36,147 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d2981510-59e6-ff69-efd1-5906a8a92785/cluster_a0b2892d-6b1d-843d-7c12-ddaf29516f86/data/data2/current/BP-1977883270-172.17.0.2-1731875758461 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T20:36:36,147 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-17T20:36:36,152 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7c6cb23c{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-17T20:36:36,153 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@38a385f1{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-17T20:36:36,153 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-17T20:36:36,153 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2a99c341{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-17T20:36:36,153 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@179d1ca6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d2981510-59e6-ff69-efd1-5906a8a92785/hadoop.log.dir/,STOPPED}
2024-11-17T20:36:36,160 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-17T20:36:36,180 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-17T20:36:36,188 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnPipelineRestart Thread=183 (was 158)
Potentially hanging thread: nioEventLoopGroup-30-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-31-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:46233
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.4@localhost:46233
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-31-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-33-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:46233 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-33-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-10-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:46233
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-32-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:46233 from jenkins.hfs.4
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:46233
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-10-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-31-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-30-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-33-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-30-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:46233 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-32-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-32-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:46233
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-10-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=457 (was 438) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=210 (was 221), ProcessCount=11 (was 11), AvailableMemoryMB=7578 (was 7767)
2024-11-17T20:36:36,195 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testCompactionRecordDoesntBlockRolling Thread=183, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=210, ProcessCount=11, AvailableMemoryMB=7578
2024-11-17T20:36:36,196 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-17T20:36:36,196 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d2981510-59e6-ff69-efd1-5906a8a92785/hadoop.log.dir so I do NOT create it in target/test-data/f4dd0321-963a-3166-6b1a-243c9e15ca98
2024-11-17T20:36:36,196 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d2981510-59e6-ff69-efd1-5906a8a92785/hadoop.tmp.dir so I do NOT create it in target/test-data/f4dd0321-963a-3166-6b1a-243c9e15ca98
2024-11-17T20:36:36,196 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4dd0321-963a-3166-6b1a-243c9e15ca98/cluster_3c7e0f58-fc63-0ea1-4594-d31d5a66aa59, deleteOnExit=true
2024-11-17T20:36:36,196 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-17T20:36:36,196 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4dd0321-963a-3166-6b1a-243c9e15ca98/test.cache.data in system properties and HBase conf
2024-11-17T20:36:36,196 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4dd0321-963a-3166-6b1a-243c9e15ca98/hadoop.tmp.dir in system properties and HBase conf
2024-11-17T20:36:36,196 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4dd0321-963a-3166-6b1a-243c9e15ca98/hadoop.log.dir in system properties and HBase conf
2024-11-17T20:36:36,197 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4dd0321-963a-3166-6b1a-243c9e15ca98/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-17T20:36:36,197 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4dd0321-963a-3166-6b1a-243c9e15ca98/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-17T20:36:36,197 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-17T20:36:36,197 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-17T20:36:36,197 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4dd0321-963a-3166-6b1a-243c9e15ca98/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-17T20:36:36,197 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4dd0321-963a-3166-6b1a-243c9e15ca98/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-17T20:36:36,197 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4dd0321-963a-3166-6b1a-243c9e15ca98/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-17T20:36:36,197 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4dd0321-963a-3166-6b1a-243c9e15ca98/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-17T20:36:36,197 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4dd0321-963a-3166-6b1a-243c9e15ca98/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-17T20:36:36,198 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4dd0321-963a-3166-6b1a-243c9e15ca98/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-17T20:36:36,198 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4dd0321-963a-3166-6b1a-243c9e15ca98/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-17T20:36:36,198 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4dd0321-963a-3166-6b1a-243c9e15ca98/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-17T20:36:36,198 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4dd0321-963a-3166-6b1a-243c9e15ca98/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-17T20:36:36,198 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4dd0321-963a-3166-6b1a-243c9e15ca98/nfs.dump.dir in system properties and HBase conf
2024-11-17T20:36:36,198 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4dd0321-963a-3166-6b1a-243c9e15ca98/java.io.tmpdir in system properties and HBase conf
2024-11-17T20:36:36,198 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4dd0321-963a-3166-6b1a-243c9e15ca98/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-17T20:36:36,198 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4dd0321-963a-3166-6b1a-243c9e15ca98/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-17T20:36:36,198 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4dd0321-963a-3166-6b1a-243c9e15ca98/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-17T20:36:36,213 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-17T20:36:36,534 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-17T20:36:36,539 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-17T20:36:36,541 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-17T20:36:36,541 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-17T20:36:36,541 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-17T20:36:36,542 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-17T20:36:36,542 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@20f59884{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4dd0321-963a-3166-6b1a-243c9e15ca98/hadoop.log.dir/,AVAILABLE}
2024-11-17T20:36:36,542 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@190ad9e7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-17T20:36:36,644 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@43498b11{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4dd0321-963a-3166-6b1a-243c9e15ca98/java.io.tmpdir/jetty-localhost-43665-hadoop-hdfs-3_4_1-tests_jar-_-any-8729686548481925882/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-17T20:36:36,645 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1367dc96{HTTP/1.1, (http/1.1)}{localhost:43665}
2024-11-17T20:36:36,645 INFO  [Time-limited test {}] server.Server(415): Started @199931ms
2024-11-17T20:36:36,657 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-17T20:36:36,785 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:36,800 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:36,959 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:37,246 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-17T20:36:37,250 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-17T20:36:37,252 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-17T20:36:37,252 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-17T20:36:37,253 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-17T20:36:37,253 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4faae08f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4dd0321-963a-3166-6b1a-243c9e15ca98/hadoop.log.dir/,AVAILABLE}
2024-11-17T20:36:37,254 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7c13156e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-17T20:36:37,365 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@40527499{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4dd0321-963a-3166-6b1a-243c9e15ca98/java.io.tmpdir/jetty-localhost-42965-hadoop-hdfs-3_4_1-tests_jar-_-any-11492639783366899512/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T20:36:37,366 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@c1ed8d4{HTTP/1.1, (http/1.1)}{localhost:42965}
2024-11-17T20:36:37,366 INFO  [Time-limited test {}] server.Server(415): Started @200652ms
2024-11-17T20:36:37,367 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-17T20:36:37,445 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-17T20:36:37,449 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-17T20:36:37,450 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-17T20:36:37,450 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-17T20:36:37,450 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-17T20:36:37,450 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@34534d9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4dd0321-963a-3166-6b1a-243c9e15ca98/hadoop.log.dir/,AVAILABLE}
2024-11-17T20:36:37,451 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@14a6d451{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-17T20:36:37,556 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@c734161{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4dd0321-963a-3166-6b1a-243c9e15ca98/java.io.tmpdir/jetty-localhost-41487-hadoop-hdfs-3_4_1-tests_jar-_-any-4609741254087132605/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T20:36:37,557 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@75e789f0{HTTP/1.1, (http/1.1)}{localhost:41487}
2024-11-17T20:36:37,557 INFO  [Time-limited test {}] server.Server(415): Started @200843ms
2024-11-17T20:36:37,558 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-17T20:36:37,786 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:37,801 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:37,959 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:38,787 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:38,802 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:38,930 WARN  [Thread-1657 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4dd0321-963a-3166-6b1a-243c9e15ca98/cluster_3c7e0f58-fc63-0ea1-4594-d31d5a66aa59/data/data1/current/BP-384780574-172.17.0.2-1731875796225/current, will proceed with Du for space computation calculation, 
2024-11-17T20:36:38,932 WARN  [Thread-1658 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4dd0321-963a-3166-6b1a-243c9e15ca98/cluster_3c7e0f58-fc63-0ea1-4594-d31d5a66aa59/data/data2/current/BP-384780574-172.17.0.2-1731875796225/current, will proceed with Du for space computation calculation, 
2024-11-17T20:36:38,950 WARN  [Thread-1621 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-17T20:36:38,955 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1c4319e0b8a3378f with lease ID 0xe03086c7380419a7: Processing first storage report for DS-654168d2-1534-4509-86af-4a2d3245ce58 from datanode DatanodeRegistration(127.0.0.1:34555, datanodeUuid=a126db33-8ce9-4169-9e7d-e6315649dbbb, infoPort=46215, infoSecurePort=0, ipcPort=43093, storageInfo=lv=-57;cid=testClusterID;nsid=1031886472;c=1731875796225)
2024-11-17T20:36:38,955 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1c4319e0b8a3378f with lease ID 0xe03086c7380419a7: from storage DS-654168d2-1534-4509-86af-4a2d3245ce58 node DatanodeRegistration(127.0.0.1:34555, datanodeUuid=a126db33-8ce9-4169-9e7d-e6315649dbbb, infoPort=46215, infoSecurePort=0, ipcPort=43093, storageInfo=lv=-57;cid=testClusterID;nsid=1031886472;c=1731875796225), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-17T20:36:38,955 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1c4319e0b8a3378f with lease ID 0xe03086c7380419a7: Processing first storage report for DS-2a0b626c-968c-4d28-a85a-070bd08f25ca from datanode DatanodeRegistration(127.0.0.1:34555, datanodeUuid=a126db33-8ce9-4169-9e7d-e6315649dbbb, infoPort=46215, infoSecurePort=0, ipcPort=43093, storageInfo=lv=-57;cid=testClusterID;nsid=1031886472;c=1731875796225)
2024-11-17T20:36:38,955 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1c4319e0b8a3378f with lease ID 0xe03086c7380419a7: from storage DS-2a0b626c-968c-4d28-a85a-070bd08f25ca node DatanodeRegistration(127.0.0.1:34555, datanodeUuid=a126db33-8ce9-4169-9e7d-e6315649dbbb, infoPort=46215, infoSecurePort=0, ipcPort=43093, storageInfo=lv=-57;cid=testClusterID;nsid=1031886472;c=1731875796225), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-17T20:36:38,960 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:39,025 WARN  [Thread-1668 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4dd0321-963a-3166-6b1a-243c9e15ca98/cluster_3c7e0f58-fc63-0ea1-4594-d31d5a66aa59/data/data3/current/BP-384780574-172.17.0.2-1731875796225/current, will proceed with Du for space computation calculation, 
2024-11-17T20:36:39,025 WARN  [Thread-1669 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4dd0321-963a-3166-6b1a-243c9e15ca98/cluster_3c7e0f58-fc63-0ea1-4594-d31d5a66aa59/data/data4/current/BP-384780574-172.17.0.2-1731875796225/current, will proceed with Du for space computation calculation, 
2024-11-17T20:36:39,042 WARN  [Thread-1644 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-17T20:36:39,044 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2c56e3d85a3ce458 with lease ID 0xe03086c7380419a8: Processing first storage report for DS-fabb6b43-fe58-40df-acb7-96509dcdee30 from datanode DatanodeRegistration(127.0.0.1:37709, datanodeUuid=60ab1532-6bfd-42ae-a362-ddccdaa9a30b, infoPort=37705, infoSecurePort=0, ipcPort=42179, storageInfo=lv=-57;cid=testClusterID;nsid=1031886472;c=1731875796225)
2024-11-17T20:36:39,044 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2c56e3d85a3ce458 with lease ID 0xe03086c7380419a8: from storage DS-fabb6b43-fe58-40df-acb7-96509dcdee30 node DatanodeRegistration(127.0.0.1:37709, datanodeUuid=60ab1532-6bfd-42ae-a362-ddccdaa9a30b, infoPort=37705, infoSecurePort=0, ipcPort=42179, storageInfo=lv=-57;cid=testClusterID;nsid=1031886472;c=1731875796225), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-17T20:36:39,044 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2c56e3d85a3ce458 with lease ID 0xe03086c7380419a8: Processing first storage report for DS-acbf4c46-031d-44ad-8567-21cbc150cf3e from datanode DatanodeRegistration(127.0.0.1:37709, datanodeUuid=60ab1532-6bfd-42ae-a362-ddccdaa9a30b, infoPort=37705, infoSecurePort=0, ipcPort=42179, storageInfo=lv=-57;cid=testClusterID;nsid=1031886472;c=1731875796225)
2024-11-17T20:36:39,044 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2c56e3d85a3ce458 with lease ID 0xe03086c7380419a8: from storage DS-acbf4c46-031d-44ad-8567-21cbc150cf3e node DatanodeRegistration(127.0.0.1:37709, datanodeUuid=60ab1532-6bfd-42ae-a362-ddccdaa9a30b, infoPort=37705, infoSecurePort=0, ipcPort=42179, storageInfo=lv=-57;cid=testClusterID;nsid=1031886472;c=1731875796225), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-17T20:36:39,104 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4dd0321-963a-3166-6b1a-243c9e15ca98
2024-11-17T20:36:39,107 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4dd0321-963a-3166-6b1a-243c9e15ca98/cluster_3c7e0f58-fc63-0ea1-4594-d31d5a66aa59/zookeeper_0, clientPort=50742, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4dd0321-963a-3166-6b1a-243c9e15ca98/cluster_3c7e0f58-fc63-0ea1-4594-d31d5a66aa59/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4dd0321-963a-3166-6b1a-243c9e15ca98/cluster_3c7e0f58-fc63-0ea1-4594-d31d5a66aa59/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-17T20:36:39,109 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=50742
2024-11-17T20:36:39,109 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T20:36:39,111 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T20:36:39,120 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37709 is added to blk_1073741825_1001 (size=7)
2024-11-17T20:36:39,121 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34555 is added to blk_1073741825_1001 (size=7)
2024-11-17T20:36:39,122 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d with version=8
2024-11-17T20:36:39,122 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/hbase-staging
2024-11-17T20:36:39,124 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/0b0b0ae8de03:0 server-side Connection retries=45
2024-11-17T20:36:39,125 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-17T20:36:39,125 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-17T20:36:39,125 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-17T20:36:39,125 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-17T20:36:39,125 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-17T20:36:39,125 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-17T20:36:39,125 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-17T20:36:39,126 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36055
2024-11-17T20:36:39,127 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:36055 connecting to ZooKeeper ensemble=127.0.0.1:50742
2024-11-17T20:36:39,217 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:360550x0, quorum=127.0.0.1:50742, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-17T20:36:39,217 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36055-0x1014a837bee0000 connected
2024-11-17T20:36:39,302 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T20:36:39,304 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T20:36:39,306 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-17T20:36:39,307 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36055-0x1014a837bee0000, quorum=127.0.0.1:50742, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-17T20:36:39,308 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d, hbase.cluster.distributed=false
2024-11-17T20:36:39,308 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:36:39,309 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:36:39,309 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:36:39,310 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:36:39,310 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:36:39,310 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36055-0x1014a837bee0000, quorum=127.0.0.1:50742, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-17T20:36:39,310 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:36:39,322 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36055
2024-11-17T20:36:39,324 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36055
2024-11-17T20:36:39,328 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36055
2024-11-17T20:36:39,329 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36055
2024-11-17T20:36:39,329 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36055
2024-11-17T20:36:39,333 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:36:39,333 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:36:39,333 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:36:39,334 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:36:39,334 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:36:39,334 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:36:39,338 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:36:39,338 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:36:39,339 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:36:39,348 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/0b0b0ae8de03:0 server-side Connection retries=45
2024-11-17T20:36:39,348 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-17T20:36:39,348 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-17T20:36:39,348 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-17T20:36:39,348 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-17T20:36:39,348 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-17T20:36:39,348 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-17T20:36:39,348 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-17T20:36:39,352 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38743
2024-11-17T20:36:39,354 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38743 connecting to ZooKeeper ensemble=127.0.0.1:50742
2024-11-17T20:36:39,355 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T20:36:39,357 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T20:36:39,369 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:387430x0, quorum=127.0.0.1:50742, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-17T20:36:39,370 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:387430x0, quorum=127.0.0.1:50742, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-17T20:36:39,370 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38743-0x1014a837bee0001 connected
2024-11-17T20:36:39,370 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-17T20:36:39,372 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-17T20:36:39,373 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38743-0x1014a837bee0001, quorum=127.0.0.1:50742, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-17T20:36:39,374 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38743-0x1014a837bee0001, quorum=127.0.0.1:50742, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-17T20:36:39,374 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38743
2024-11-17T20:36:39,375 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38743
2024-11-17T20:36:39,375 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38743
2024-11-17T20:36:39,383 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38743
2024-11-17T20:36:39,383 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38743
2024-11-17T20:36:39,397 DEBUG [M:0;0b0b0ae8de03:36055 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;0b0b0ae8de03:36055
2024-11-17T20:36:39,397 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/0b0b0ae8de03,36055,1731875799124
2024-11-17T20:36:39,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36055-0x1014a837bee0000, quorum=127.0.0.1:50742, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-17T20:36:39,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38743-0x1014a837bee0001, quorum=127.0.0.1:50742, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-17T20:36:39,407 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36055-0x1014a837bee0000, quorum=127.0.0.1:50742, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/0b0b0ae8de03,36055,1731875799124
2024-11-17T20:36:39,417 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38743-0x1014a837bee0001, quorum=127.0.0.1:50742, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-17T20:36:39,417 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36055-0x1014a837bee0000, quorum=127.0.0.1:50742, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:36:39,417 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38743-0x1014a837bee0001, quorum=127.0.0.1:50742, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:36:39,418 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36055-0x1014a837bee0000, quorum=127.0.0.1:50742, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-17T20:36:39,418 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/0b0b0ae8de03,36055,1731875799124 from backup master directory
2024-11-17T20:36:39,428 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36055-0x1014a837bee0000, quorum=127.0.0.1:50742, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/0b0b0ae8de03,36055,1731875799124
2024-11-17T20:36:39,428 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38743-0x1014a837bee0001, quorum=127.0.0.1:50742, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-17T20:36:39,428 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36055-0x1014a837bee0000, quorum=127.0.0.1:50742, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-17T20:36:39,428 WARN  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-17T20:36:39,428 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=0b0b0ae8de03,36055,1731875799124
2024-11-17T20:36:39,432 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/hbase.id] with ID: 681319f3-af2a-4446-8022-b87bc9531c4b
2024-11-17T20:36:39,432 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/.tmp/hbase.id
2024-11-17T20:36:39,439 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37709 is added to blk_1073741826_1002 (size=42)
2024-11-17T20:36:39,440 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34555 is added to blk_1073741826_1002 (size=42)
2024-11-17T20:36:39,440 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/.tmp/hbase.id]:[hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/hbase.id]
2024-11-17T20:36:39,453 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T20:36:39,453 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-17T20:36:39,455 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms.
2024-11-17T20:36:39,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36055-0x1014a837bee0000, quorum=127.0.0.1:50742, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:36:39,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38743-0x1014a837bee0001, quorum=127.0.0.1:50742, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:36:39,470 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34555 is added to blk_1073741827_1003 (size=196)
2024-11-17T20:36:39,470 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37709 is added to blk_1073741827_1003 (size=196)
2024-11-17T20:36:39,471 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-17T20:36:39,471 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-17T20:36:39,472 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-17T20:36:39,478 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37709 is added to blk_1073741828_1004 (size=1189)
2024-11-17T20:36:39,478 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34555 is added to blk_1073741828_1004 (size=1189)
2024-11-17T20:36:39,479 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/MasterData/data/master/store
2024-11-17T20:36:39,486 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37709 is added to blk_1073741829_1005 (size=34)
2024-11-17T20:36:39,486 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34555 is added to blk_1073741829_1005 (size=34)
2024-11-17T20:36:39,487 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T20:36:39,487 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-17T20:36:39,487 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T20:36:39,487 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T20:36:39,487 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-17T20:36:39,487 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T20:36:39,487 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T20:36:39,487 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731875799487Disabling compacts and flushes for region at 1731875799487Disabling writes for close at 1731875799487Writing region close event to WAL at 1731875799487Closed at 1731875799487
2024-11-17T20:36:39,488 WARN  [master/0b0b0ae8de03:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/MasterData/data/master/store/.initializing
2024-11-17T20:36:39,488 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/MasterData/WALs/0b0b0ae8de03,36055,1731875799124
2024-11-17T20:36:39,490 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0b0b0ae8de03%2C36055%2C1731875799124, suffix=, logDir=hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/MasterData/WALs/0b0b0ae8de03,36055,1731875799124, archiveDir=hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/MasterData/oldWALs, maxLogs=10
2024-11-17T20:36:39,490 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0b0b0ae8de03%2C36055%2C1731875799124.1731875799490
2024-11-17T20:36:39,495 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/MasterData/WALs/0b0b0ae8de03,36055,1731875799124/0b0b0ae8de03%2C36055%2C1731875799124.1731875799490
2024-11-17T20:36:39,504 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37705:37705),(127.0.0.1/127.0.0.1:46215:46215)]
2024-11-17T20:36:39,505 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-17T20:36:39,505 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T20:36:39,505 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T20:36:39,505 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T20:36:39,507 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-17T20:36:39,508 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-17T20:36:39,508 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:36:39,508 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T20:36:39,508 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-17T20:36:39,510 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-17T20:36:39,510 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:36:39,510 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-17T20:36:39,510 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-17T20:36:39,511 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-17T20:36:39,511 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:36:39,512 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-17T20:36:39,512 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-17T20:36:39,513 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-17T20:36:39,513 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:36:39,514 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-17T20:36:39,514 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T20:36:39,514 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-17T20:36:39,515 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-17T20:36:39,516 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T20:36:39,516 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T20:36:39,517 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-17T20:36:39,518 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T20:36:39,520 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-17T20:36:39,520 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=790997, jitterRate=0.005805224180221558}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-17T20:36:39,521 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1731875799505Initializing all the Stores at 1731875799506 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731875799506Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731875799506Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731875799506Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731875799506Cleaning up temporary data from old regions at 1731875799516 (+10 ms)Region opened successfully at 1731875799521 (+5 ms)
2024-11-17T20:36:39,521 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-17T20:36:39,524 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64232261, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0b0b0ae8de03/172.17.0.2:0
2024-11-17T20:36:39,525 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-17T20:36:39,525 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-17T20:36:39,525 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-17T20:36:39,525 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-17T20:36:39,526 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-17T20:36:39,526 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-17T20:36:39,526 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-17T20:36:39,528 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-17T20:36:39,529 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36055-0x1014a837bee0000, quorum=127.0.0.1:50742, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-17T20:36:39,537 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-17T20:36:39,538 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-17T20:36:39,539 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36055-0x1014a837bee0000, quorum=127.0.0.1:50742, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-17T20:36:39,548 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-17T20:36:39,548 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-17T20:36:39,550 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36055-0x1014a837bee0000, quorum=127.0.0.1:50742, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-17T20:36:39,558 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-17T20:36:39,559 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36055-0x1014a837bee0000, quorum=127.0.0.1:50742, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-17T20:36:39,569 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-17T20:36:39,571 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36055-0x1014a837bee0000, quorum=127.0.0.1:50742, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-17T20:36:39,579 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-17T20:36:39,590 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36055-0x1014a837bee0000, quorum=127.0.0.1:50742, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-17T20:36:39,590 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38743-0x1014a837bee0001, quorum=127.0.0.1:50742, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-17T20:36:39,590 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36055-0x1014a837bee0000, quorum=127.0.0.1:50742, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:36:39,590 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38743-0x1014a837bee0001, quorum=127.0.0.1:50742, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:36:39,591 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=0b0b0ae8de03,36055,1731875799124, sessionid=0x1014a837bee0000, setting cluster-up flag (Was=false)
2024-11-17T20:36:39,611 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36055-0x1014a837bee0000, quorum=127.0.0.1:50742, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:36:39,611 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38743-0x1014a837bee0001, quorum=127.0.0.1:50742, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:36:39,643 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-17T20:36:39,644 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0b0b0ae8de03,36055,1731875799124
2024-11-17T20:36:39,664 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36055-0x1014a837bee0000, quorum=127.0.0.1:50742, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:36:39,664 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38743-0x1014a837bee0001, quorum=127.0.0.1:50742, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:36:39,695 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-17T20:36:39,697 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0b0b0ae8de03,36055,1731875799124
2024-11-17T20:36:39,698 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-17T20:36:39,700 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-17T20:36:39,701 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-17T20:36:39,701 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-17T20:36:39,701 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 0b0b0ae8de03,36055,1731875799124
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-17T20:36:39,703 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/0b0b0ae8de03:0, corePoolSize=5, maxPoolSize=5
2024-11-17T20:36:39,703 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/0b0b0ae8de03:0, corePoolSize=5, maxPoolSize=5
2024-11-17T20:36:39,704 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/0b0b0ae8de03:0, corePoolSize=5, maxPoolSize=5
2024-11-17T20:36:39,704 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/0b0b0ae8de03:0, corePoolSize=5, maxPoolSize=5
2024-11-17T20:36:39,704 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/0b0b0ae8de03:0, corePoolSize=10, maxPoolSize=10
2024-11-17T20:36:39,704 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:36:39,704 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/0b0b0ae8de03:0, corePoolSize=2, maxPoolSize=2
2024-11-17T20:36:39,704 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:36:39,705 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731875829705
2024-11-17T20:36:39,705 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-17T20:36:39,705 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-17T20:36:39,705 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-17T20:36:39,705 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-17T20:36:39,705 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-17T20:36:39,705 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-17T20:36:39,706 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-17T20:36:39,706 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-17T20:36:39,706 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-17T20:36:39,706 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-17T20:36:39,706 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-17T20:36:39,706 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-17T20:36:39,707 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-17T20:36:39,707 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-17T20:36:39,707 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/0b0b0ae8de03:0:becomeActiveMaster-HFileCleaner.large.0-1731875799707,5,FailOnTimeoutGroup]
2024-11-17T20:36:39,707 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/0b0b0ae8de03:0:becomeActiveMaster-HFileCleaner.small.0-1731875799707,5,FailOnTimeoutGroup]
2024-11-17T20:36:39,707 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-17T20:36:39,707 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-17T20:36:39,707 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-17T20:36:39,707 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-17T20:36:39,708 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:36:39,708 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-17T20:36:39,720 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34555 is added to blk_1073741831_1007 (size=1321)
2024-11-17T20:36:39,720 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37709 is added to blk_1073741831_1007 (size=1321)
2024-11-17T20:36:39,721 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-17T20:36:39,722 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d
2024-11-17T20:36:39,727 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34555 is added to blk_1073741832_1008 (size=32)
2024-11-17T20:36:39,727 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37709 is added to blk_1073741832_1008 (size=32)
2024-11-17T20:36:39,728 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T20:36:39,729 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-17T20:36:39,730 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-17T20:36:39,730 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:36:39,731 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T20:36:39,731 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-17T20:36:39,732 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-17T20:36:39,732 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:36:39,733 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T20:36:39,733 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-17T20:36:39,734 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-17T20:36:39,734 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:36:39,735 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T20:36:39,735 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-17T20:36:39,736 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-17T20:36:39,736 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:36:39,737 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T20:36:39,737 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-17T20:36:39,738 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/data/hbase/meta/1588230740
2024-11-17T20:36:39,738 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/data/hbase/meta/1588230740
2024-11-17T20:36:39,739 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-17T20:36:39,740 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-17T20:36:39,740 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-17T20:36:39,741 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-17T20:36:39,743 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-17T20:36:39,744 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=793285, jitterRate=0.008714422583580017}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-17T20:36:39,745 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1731875799728Initializing all the Stores at 1731875799729 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731875799729Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731875799729Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731875799729Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731875799729Cleaning up temporary data from old regions at 1731875799740 (+11 ms)Region opened successfully at 1731875799745 (+5 ms)
2024-11-17T20:36:39,745 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-17T20:36:39,745 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-17T20:36:39,745 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-17T20:36:39,745 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-17T20:36:39,745 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-17T20:36:39,745 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-17T20:36:39,745 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731875799745Disabling compacts and flushes for region at 1731875799745Disabling writes for close at 1731875799745Writing region close event to WAL at 1731875799745Closed at 1731875799745
2024-11-17T20:36:39,746 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-17T20:36:39,747 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-17T20:36:39,747 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-17T20:36:39,748 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-17T20:36:39,749 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-17T20:36:39,785 INFO  [RS:0;0b0b0ae8de03:38743 {}] regionserver.HRegionServer(746): ClusterId : 681319f3-af2a-4446-8022-b87bc9531c4b
2024-11-17T20:36:39,785 DEBUG [RS:0;0b0b0ae8de03:38743 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-17T20:36:39,788 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:39,800 DEBUG [RS:0;0b0b0ae8de03:38743 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-17T20:36:39,800 DEBUG [RS:0;0b0b0ae8de03:38743 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-17T20:36:39,802 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:39,812 DEBUG [RS:0;0b0b0ae8de03:38743 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-17T20:36:39,812 DEBUG [RS:0;0b0b0ae8de03:38743 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f9cc52d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0b0b0ae8de03/172.17.0.2:0
2024-11-17T20:36:39,826 DEBUG [RS:0;0b0b0ae8de03:38743 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;0b0b0ae8de03:38743
2024-11-17T20:36:39,826 INFO  [RS:0;0b0b0ae8de03:38743 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-17T20:36:39,826 INFO  [RS:0;0b0b0ae8de03:38743 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-17T20:36:39,826 DEBUG [RS:0;0b0b0ae8de03:38743 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-17T20:36:39,827 INFO  [RS:0;0b0b0ae8de03:38743 {}] regionserver.HRegionServer(2659): reportForDuty to master=0b0b0ae8de03,36055,1731875799124 with port=38743, startcode=1731875799347
2024-11-17T20:36:39,827 DEBUG [RS:0;0b0b0ae8de03:38743 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-17T20:36:39,828 INFO  [HMaster-EventLoopGroup-12-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53929, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.5 (auth:SIMPLE), service=RegionServerStatusService
2024-11-17T20:36:39,829 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36055 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 0b0b0ae8de03,38743,1731875799347
2024-11-17T20:36:39,829 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36055 {}] master.ServerManager(517): Registering regionserver=0b0b0ae8de03,38743,1731875799347
2024-11-17T20:36:39,831 DEBUG [RS:0;0b0b0ae8de03:38743 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d
2024-11-17T20:36:39,831 DEBUG [RS:0;0b0b0ae8de03:38743 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:42353
2024-11-17T20:36:39,831 DEBUG [RS:0;0b0b0ae8de03:38743 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-17T20:36:39,843 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36055-0x1014a837bee0000, quorum=127.0.0.1:50742, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-17T20:36:39,843 DEBUG [RS:0;0b0b0ae8de03:38743 {}] zookeeper.ZKUtil(111): regionserver:38743-0x1014a837bee0001, quorum=127.0.0.1:50742, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/0b0b0ae8de03,38743,1731875799347
2024-11-17T20:36:39,843 WARN  [RS:0;0b0b0ae8de03:38743 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-17T20:36:39,843 INFO  [RS:0;0b0b0ae8de03:38743 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-17T20:36:39,843 DEBUG [RS:0;0b0b0ae8de03:38743 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/WALs/0b0b0ae8de03,38743,1731875799347
2024-11-17T20:36:39,844 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [0b0b0ae8de03,38743,1731875799347]
2024-11-17T20:36:39,847 INFO  [RS:0;0b0b0ae8de03:38743 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-17T20:36:39,849 INFO  [RS:0;0b0b0ae8de03:38743 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-17T20:36:39,849 INFO  [RS:0;0b0b0ae8de03:38743 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-17T20:36:39,849 INFO  [RS:0;0b0b0ae8de03:38743 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-17T20:36:39,849 INFO  [RS:0;0b0b0ae8de03:38743 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-17T20:36:39,850 INFO  [RS:0;0b0b0ae8de03:38743 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-17T20:36:39,850 INFO  [RS:0;0b0b0ae8de03:38743 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-17T20:36:39,850 DEBUG [RS:0;0b0b0ae8de03:38743 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:36:39,850 DEBUG [RS:0;0b0b0ae8de03:38743 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:36:39,850 DEBUG [RS:0;0b0b0ae8de03:38743 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:36:39,850 DEBUG [RS:0;0b0b0ae8de03:38743 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:36:39,850 DEBUG [RS:0;0b0b0ae8de03:38743 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:36:39,850 DEBUG [RS:0;0b0b0ae8de03:38743 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/0b0b0ae8de03:0, corePoolSize=2, maxPoolSize=2
2024-11-17T20:36:39,851 DEBUG [RS:0;0b0b0ae8de03:38743 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:36:39,851 DEBUG [RS:0;0b0b0ae8de03:38743 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:36:39,851 DEBUG [RS:0;0b0b0ae8de03:38743 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:36:39,851 DEBUG [RS:0;0b0b0ae8de03:38743 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:36:39,851 DEBUG [RS:0;0b0b0ae8de03:38743 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:36:39,851 DEBUG [RS:0;0b0b0ae8de03:38743 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:36:39,851 DEBUG [RS:0;0b0b0ae8de03:38743 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/0b0b0ae8de03:0, corePoolSize=3, maxPoolSize=3
2024-11-17T20:36:39,851 DEBUG [RS:0;0b0b0ae8de03:38743 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0, corePoolSize=3, maxPoolSize=3
2024-11-17T20:36:39,851 INFO  [RS:0;0b0b0ae8de03:38743 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-17T20:36:39,851 INFO  [RS:0;0b0b0ae8de03:38743 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-17T20:36:39,851 INFO  [RS:0;0b0b0ae8de03:38743 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-17T20:36:39,852 INFO  [RS:0;0b0b0ae8de03:38743 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-17T20:36:39,852 INFO  [RS:0;0b0b0ae8de03:38743 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-17T20:36:39,852 INFO  [RS:0;0b0b0ae8de03:38743 {}] hbase.ChoreService(168): Chore ScheduledChore name=0b0b0ae8de03,38743,1731875799347-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-17T20:36:39,868 INFO  [RS:0;0b0b0ae8de03:38743 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-17T20:36:39,868 INFO  [RS:0;0b0b0ae8de03:38743 {}] hbase.ChoreService(168): Chore ScheduledChore name=0b0b0ae8de03,38743,1731875799347-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-17T20:36:39,868 INFO  [RS:0;0b0b0ae8de03:38743 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T20:36:39,868 INFO  [RS:0;0b0b0ae8de03:38743 {}] regionserver.Replication(171): 0b0b0ae8de03,38743,1731875799347 started
2024-11-17T20:36:39,884 INFO  [RS:0;0b0b0ae8de03:38743 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T20:36:39,884 INFO  [RS:0;0b0b0ae8de03:38743 {}] regionserver.HRegionServer(1482): Serving as 0b0b0ae8de03,38743,1731875799347, RpcServer on 0b0b0ae8de03/172.17.0.2:38743, sessionid=0x1014a837bee0001
2024-11-17T20:36:39,884 DEBUG [RS:0;0b0b0ae8de03:38743 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-17T20:36:39,884 DEBUG [RS:0;0b0b0ae8de03:38743 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 0b0b0ae8de03,38743,1731875799347
2024-11-17T20:36:39,884 DEBUG [RS:0;0b0b0ae8de03:38743 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0b0b0ae8de03,38743,1731875799347'
2024-11-17T20:36:39,884 DEBUG [RS:0;0b0b0ae8de03:38743 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-17T20:36:39,885 DEBUG [RS:0;0b0b0ae8de03:38743 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-17T20:36:39,885 DEBUG [RS:0;0b0b0ae8de03:38743 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-17T20:36:39,885 DEBUG [RS:0;0b0b0ae8de03:38743 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-17T20:36:39,885 DEBUG [RS:0;0b0b0ae8de03:38743 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 0b0b0ae8de03,38743,1731875799347
2024-11-17T20:36:39,885 DEBUG [RS:0;0b0b0ae8de03:38743 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0b0b0ae8de03,38743,1731875799347'
2024-11-17T20:36:39,885 DEBUG [RS:0;0b0b0ae8de03:38743 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-17T20:36:39,886 DEBUG [RS:0;0b0b0ae8de03:38743 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-17T20:36:39,886 DEBUG [RS:0;0b0b0ae8de03:38743 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-17T20:36:39,886 INFO  [RS:0;0b0b0ae8de03:38743 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-17T20:36:39,886 INFO  [RS:0;0b0b0ae8de03:38743 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-17T20:36:39,899 WARN  [0b0b0ae8de03:36055 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions.
2024-11-17T20:36:39,961 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:39,988 INFO  [RS:0;0b0b0ae8de03:38743 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0b0b0ae8de03%2C38743%2C1731875799347, suffix=, logDir=hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/WALs/0b0b0ae8de03,38743,1731875799347, archiveDir=hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/oldWALs, maxLogs=32
2024-11-17T20:36:39,988 INFO  [RS:0;0b0b0ae8de03:38743 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0b0b0ae8de03%2C38743%2C1731875799347.1731875799988
2024-11-17T20:36:39,993 INFO  [RS:0;0b0b0ae8de03:38743 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/WALs/0b0b0ae8de03,38743,1731875799347/0b0b0ae8de03%2C38743%2C1731875799347.1731875799988
2024-11-17T20:36:39,994 DEBUG [RS:0;0b0b0ae8de03:38743 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37705:37705),(127.0.0.1/127.0.0.1:46215:46215)]
2024-11-17T20:36:40,150 DEBUG [0b0b0ae8de03:36055 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-17T20:36:40,150 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=0b0b0ae8de03,38743,1731875799347
2024-11-17T20:36:40,152 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0b0b0ae8de03,38743,1731875799347, state=OPENING
2024-11-17T20:36:40,164 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-17T20:36:40,175 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38743-0x1014a837bee0001, quorum=127.0.0.1:50742, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:36:40,175 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36055-0x1014a837bee0000, quorum=127.0.0.1:50742, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:36:40,176 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-17T20:36:40,176 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-17T20:36:40,176 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-17T20:36:40,176 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=0b0b0ae8de03,38743,1731875799347}]
2024-11-17T20:36:40,331 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-17T20:36:40,333 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35465, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-17T20:36:40,336 INFO  [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-17T20:36:40,337 INFO  [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-17T20:36:40,339 INFO  [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0b0b0ae8de03%2C38743%2C1731875799347.meta, suffix=.meta, logDir=hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/WALs/0b0b0ae8de03,38743,1731875799347, archiveDir=hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/oldWALs, maxLogs=32
2024-11-17T20:36:40,339 INFO  [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 0b0b0ae8de03%2C38743%2C1731875799347.meta.1731875800339.meta
2024-11-17T20:36:40,344 INFO  [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/WALs/0b0b0ae8de03,38743,1731875799347/0b0b0ae8de03%2C38743%2C1731875799347.meta.1731875800339.meta
2024-11-17T20:36:40,351 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37705:37705),(127.0.0.1/127.0.0.1:46215:46215)]
2024-11-17T20:36:40,352 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-17T20:36:40,352 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-17T20:36:40,352 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-17T20:36:40,353 INFO  [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-17T20:36:40,353 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-17T20:36:40,353 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T20:36:40,353 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-17T20:36:40,353 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-17T20:36:40,354 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-17T20:36:40,355 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-17T20:36:40,355 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:36:40,356 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T20:36:40,356 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-17T20:36:40,356 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-17T20:36:40,356 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:36:40,357 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T20:36:40,357 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-17T20:36:40,358 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-17T20:36:40,358 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:36:40,358 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T20:36:40,358 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-17T20:36:40,359 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-17T20:36:40,359 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:36:40,359 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T20:36:40,359 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-17T20:36:40,360 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/data/hbase/meta/1588230740
2024-11-17T20:36:40,361 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/data/hbase/meta/1588230740
2024-11-17T20:36:40,362 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-17T20:36:40,362 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-17T20:36:40,362 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-17T20:36:40,364 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-17T20:36:40,364 INFO  [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=789531, jitterRate=0.003941446542739868}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-17T20:36:40,365 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-17T20:36:40,365 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1731875800353Writing region info on filesystem at 1731875800353Initializing all the Stores at 1731875800354 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731875800354Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731875800354Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731875800354Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731875800354Cleaning up temporary data from old regions at 1731875800362 (+8 ms)Running coprocessor post-open hooks at 1731875800365 (+3 ms)Region opened successfully at 1731875800365
2024-11-17T20:36:40,366 INFO  [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731875800331
2024-11-17T20:36:40,369 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-17T20:36:40,369 INFO  [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-17T20:36:40,370 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=0b0b0ae8de03,38743,1731875799347
2024-11-17T20:36:40,371 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0b0b0ae8de03,38743,1731875799347, state=OPEN
2024-11-17T20:36:40,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36055-0x1014a837bee0000, quorum=127.0.0.1:50742, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-17T20:36:40,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38743-0x1014a837bee0001, quorum=127.0.0.1:50742, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-17T20:36:40,407 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-17T20:36:40,407 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-17T20:36:40,407 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=0b0b0ae8de03,38743,1731875799347
2024-11-17T20:36:40,412 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-17T20:36:40,413 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=0b0b0ae8de03,38743,1731875799347 in 231 msec
2024-11-17T20:36:40,417 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-17T20:36:40,417 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 666 msec
2024-11-17T20:36:40,418 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-17T20:36:40,418 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-17T20:36:40,419 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-17T20:36:40,419 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,38743,1731875799347, seqNum=-1]
2024-11-17T20:36:40,419 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-17T20:36:40,421 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-13-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53295, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-17T20:36:40,426 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 725 msec
2024-11-17T20:36:40,426 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731875800426, completionTime=-1
2024-11-17T20:36:40,426 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-17T20:36:40,427 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster...
2024-11-17T20:36:40,429 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1
2024-11-17T20:36:40,429 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731875860429
2024-11-17T20:36:40,429 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731875920429
2024-11-17T20:36:40,429 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 2 msec
2024-11-17T20:36:40,429 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0b0b0ae8de03,36055,1731875799124-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-17T20:36:40,430 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0b0b0ae8de03,36055,1731875799124-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T20:36:40,430 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0b0b0ae8de03,36055,1731875799124-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T20:36:40,430 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-0b0b0ae8de03:36055, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T20:36:40,430 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-17T20:36:40,431 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-17T20:36:40,432 DEBUG [master/0b0b0ae8de03:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-17T20:36:40,435 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.006sec
2024-11-17T20:36:40,435 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-17T20:36:40,435 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-17T20:36:40,435 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-17T20:36:40,435 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-17T20:36:40,435 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-17T20:36:40,435 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0b0b0ae8de03,36055,1731875799124-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-17T20:36:40,435 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0b0b0ae8de03,36055,1731875799124-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-17T20:36:40,437 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-17T20:36:40,437 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-17T20:36:40,437 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0b0b0ae8de03,36055,1731875799124-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T20:36:40,486 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62ff6bde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-17T20:36:40,486 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b0b0ae8de03,36055,-1 for getting cluster id
2024-11-17T20:36:40,486 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-17T20:36:40,489 DEBUG [HMaster-EventLoopGroup-12-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '681319f3-af2a-4446-8022-b87bc9531c4b'
2024-11-17T20:36:40,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-17T20:36:40,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "681319f3-af2a-4446-8022-b87bc9531c4b"

2024-11-17T20:36:40,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fadd8a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-17T20:36:40,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b0b0ae8de03,36055,-1]
2024-11-17T20:36:40,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-17T20:36:40,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T20:36:40,493 INFO  [HMaster-EventLoopGroup-12-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39806, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-17T20:36:40,494 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@550e9c9b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-17T20:36:40,495 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-17T20:36:40,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,38743,1731875799347, seqNum=-1]
2024-11-17T20:36:40,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-17T20:36:40,498 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53542, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-17T20:36:40,500 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=0b0b0ae8de03,36055,1731875799124
2024-11-17T20:36:40,500 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T20:36:40,503 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-17T20:36:40,504 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-17T20:36:40,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncConnectionImpl(321): The fetched master address is 0b0b0ae8de03,36055,1731875799124
2024-11-17T20:36:40,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@158cc672
2024-11-17T20:36:40,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-17T20:36:40,506 INFO  [HMaster-EventLoopGroup-12-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39810, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-17T20:36:40,507 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36055 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-11-17T20:36:40,507 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36055 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-11-17T20:36:40,507 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36055 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testCompactionRecordDoesntBlockRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-17T20:36:40,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36055 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-17T20:36:40,509 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-11-17T20:36:40,510 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:36:40,510 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36055 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testCompactionRecordDoesntBlockRolling"
 procId is: 4
2024-11-17T20:36:40,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-17T20:36:40,511 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-17T20:36:40,527 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34555 is added to blk_1073741835_1011 (size=405)
2024-11-17T20:36:40,527 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37709 is added to blk_1073741835_1011 (size=405)
2024-11-17T20:36:40,529 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 30bad7b694a399a1221a097a4c1c0c96, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731875800506.30bad7b694a399a1221a097a4c1c0c96.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testCompactionRecordDoesntBlockRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d
2024-11-17T20:36:40,535 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37709 is added to blk_1073741836_1012 (size=88)
2024-11-17T20:36:40,535 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34555 is added to blk_1073741836_1012 (size=88)
2024-11-17T20:36:40,535 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731875800506.30bad7b694a399a1221a097a4c1c0c96.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T20:36:40,536 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1722): Closing 30bad7b694a399a1221a097a4c1c0c96, disabling compactions & flushes
2024-11-17T20:36:40,536 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731875800506.30bad7b694a399a1221a097a4c1c0c96.
2024-11-17T20:36:40,536 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731875800506.30bad7b694a399a1221a097a4c1c0c96.
2024-11-17T20:36:40,536 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731875800506.30bad7b694a399a1221a097a4c1c0c96. after waiting 0 ms
2024-11-17T20:36:40,536 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731875800506.30bad7b694a399a1221a097a4c1c0c96.
2024-11-17T20:36:40,536 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731875800506.30bad7b694a399a1221a097a4c1c0c96.
2024-11-17T20:36:40,536 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for 30bad7b694a399a1221a097a4c1c0c96:
Waiting for close lock at 1731875800535Disabling compacts and flushes for region at 1731875800535Disabling writes for close at 1731875800536 (+1 ms)Writing region close event to WAL at 1731875800536Closed at 1731875800536
2024-11-17T20:36:40,537 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ADD_TO_META
2024-11-17T20:36:40,537 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731875800506.30bad7b694a399a1221a097a4c1c0c96.","families":{"info":[{"qualifier":"regioninfo","vlen":87,"tag":[],"timestamp":"1731875800537"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731875800537"}]},"ts":"1731875800537"}
2024-11-17T20:36:40,540 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-17T20:36:40,541 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-17T20:36:40,542 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731875800541"}]},"ts":"1731875800541"}
2024-11-17T20:36:40,544 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLING in hbase:meta
2024-11-17T20:36:40,544 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=30bad7b694a399a1221a097a4c1c0c96, ASSIGN}]
2024-11-17T20:36:40,546 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=30bad7b694a399a1221a097a4c1c0c96, ASSIGN
2024-11-17T20:36:40,547 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=30bad7b694a399a1221a097a4c1c0c96, ASSIGN; state=OFFLINE, location=0b0b0ae8de03,38743,1731875799347; forceNewPlan=false, retain=false
2024-11-17T20:36:40,698 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=30bad7b694a399a1221a097a4c1c0c96, regionState=OPENING, regionLocation=0b0b0ae8de03,38743,1731875799347
2024-11-17T20:36:40,700 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-13-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=30bad7b694a399a1221a097a4c1c0c96, ASSIGN because future has completed
2024-11-17T20:36:40,701 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 30bad7b694a399a1221a097a4c1c0c96, server=0b0b0ae8de03,38743,1731875799347}]
2024-11-17T20:36:40,788 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:40,803 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:40,858 INFO  [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731875800506.30bad7b694a399a1221a097a4c1c0c96.
2024-11-17T20:36:40,858 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 30bad7b694a399a1221a097a4c1c0c96, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731875800506.30bad7b694a399a1221a097a4c1c0c96.', STARTKEY => '', ENDKEY => ''}
2024-11-17T20:36:40,859 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testCompactionRecordDoesntBlockRolling 30bad7b694a399a1221a097a4c1c0c96
2024-11-17T20:36:40,859 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731875800506.30bad7b694a399a1221a097a4c1c0c96.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T20:36:40,859 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 30bad7b694a399a1221a097a4c1c0c96
2024-11-17T20:36:40,859 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 30bad7b694a399a1221a097a4c1c0c96
2024-11-17T20:36:40,861 INFO  [StoreOpener-30bad7b694a399a1221a097a4c1c0c96-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 30bad7b694a399a1221a097a4c1c0c96 
2024-11-17T20:36:40,862 INFO  [StoreOpener-30bad7b694a399a1221a097a4c1c0c96-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 30bad7b694a399a1221a097a4c1c0c96 columnFamilyName info
2024-11-17T20:36:40,862 DEBUG [StoreOpener-30bad7b694a399a1221a097a4c1c0c96-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:36:40,863 INFO  [StoreOpener-30bad7b694a399a1221a097a4c1c0c96-1 {}] regionserver.HStore(327): Store=30bad7b694a399a1221a097a4c1c0c96/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-17T20:36:40,863 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 30bad7b694a399a1221a097a4c1c0c96
2024-11-17T20:36:40,864 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/30bad7b694a399a1221a097a4c1c0c96
2024-11-17T20:36:40,864 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/30bad7b694a399a1221a097a4c1c0c96
2024-11-17T20:36:40,865 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 30bad7b694a399a1221a097a4c1c0c96
2024-11-17T20:36:40,865 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 30bad7b694a399a1221a097a4c1c0c96
2024-11-17T20:36:40,867 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 30bad7b694a399a1221a097a4c1c0c96
2024-11-17T20:36:40,870 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/30bad7b694a399a1221a097a4c1c0c96/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-17T20:36:40,870 INFO  [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 30bad7b694a399a1221a097a4c1c0c96; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=806728, jitterRate=0.025808140635490417}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-17T20:36:40,870 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 30bad7b694a399a1221a097a4c1c0c96
2024-11-17T20:36:40,871 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 30bad7b694a399a1221a097a4c1c0c96:
Running coprocessor pre-open hook at 1731875800859Writing region info on filesystem at 1731875800859Initializing all the Stores at 1731875800860 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731875800860Cleaning up temporary data from old regions at 1731875800865 (+5 ms)Running coprocessor post-open hooks at 1731875800870 (+5 ms)Region opened successfully at 1731875800871 (+1 ms)
2024-11-17T20:36:40,872 INFO  [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731875800506.30bad7b694a399a1221a097a4c1c0c96., pid=6, masterSystemTime=1731875800854
2024-11-17T20:36:40,874 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731875800506.30bad7b694a399a1221a097a4c1c0c96.
2024-11-17T20:36:40,874 INFO  [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731875800506.30bad7b694a399a1221a097a4c1c0c96.
2024-11-17T20:36:40,875 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=30bad7b694a399a1221a097a4c1c0c96, regionState=OPEN, openSeqNum=2, regionLocation=0b0b0ae8de03,38743,1731875799347
2024-11-17T20:36:40,877 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-13-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 30bad7b694a399a1221a097a4c1c0c96, server=0b0b0ae8de03,38743,1731875799347 because future has completed
2024-11-17T20:36:40,881 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-17T20:36:40,881 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 30bad7b694a399a1221a097a4c1c0c96, server=0b0b0ae8de03,38743,1731875799347 in 178 msec
2024-11-17T20:36:40,884 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-17T20:36:40,884 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=30bad7b694a399a1221a097a4c1c0c96, ASSIGN in 337 msec
2024-11-17T20:36:40,885 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-17T20:36:40,885 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731875800885"}]},"ts":"1731875800885"}
2024-11-17T20:36:40,887 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLED in hbase:meta
2024-11-17T20:36:40,888 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_POST_OPERATION
2024-11-17T20:36:40,890 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 381 msec
2024-11-17T20:36:40,961 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:41,789 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:41,803 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:41,962 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:42,790 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:42,804 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:42,963 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:43,791 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:43,803 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-17T20:36:43,803 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-11-17T20:36:43,804 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-17T20:36:43,804 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-11-17T20:36:43,805 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-11-17T20:36:43,805 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer
2024-11-17T20:36:43,805 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-17T20:36:43,805 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling Metrics about Tables on a single HBase RegionServer
2024-11-17T20:36:43,805 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:43,963 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:44,791 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:44,806 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:44,964 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:45,792 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:45,806 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:45,856 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-17T20:36:45,857 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:36:45,858 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:36:45,858 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:36:45,858 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:36:45,859 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:36:45,859 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:36:45,885 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:36:45,885 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:36:45,885 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:36:45,886 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:36:45,886 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:36:45,886 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:36:45,891 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:36:45,891 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:36:45,891 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:36:45,900 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-17T20:36:45,901 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testCompactionRecordDoesntBlockRolling'
2024-11-17T20:36:45,965 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:45,965 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650 after 68047ms
java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor204.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:36:46,793 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:46,807 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:46,965 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:47,794 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:47,808 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:47,966 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:48,795 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:48,808 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:48,967 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:49,795 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:49,809 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:49,967 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:50,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-17T20:36:50,558 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-11-17T20:36:50,558 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testCompactionRecordDoesntBlockRolling,, stopping at row=TestLogRolling-testCompactionRecordDoesntBlockRolling ,, for max=2147483647 with caching=100
2024-11-17T20:36:50,563 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-17T20:36:50,563 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731875800506.30bad7b694a399a1221a097a4c1c0c96.
2024-11-17T20:36:50,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testCompactionRecordDoesntBlockRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731875800506.30bad7b694a399a1221a097a4c1c0c96., hostname=0b0b0ae8de03,38743,1731875799347, seqNum=2]
2024-11-17T20:36:50,575 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36055 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-17T20:36:50,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36055 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-17T20:36:50,581 INFO  [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-11-17T20:36:50,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7
2024-11-17T20:36:50,583 INFO  [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-11-17T20:36:50,584 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-11-17T20:36:50,745 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38743 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8
2024-11-17T20:36:50,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731875800506.30bad7b694a399a1221a097a4c1c0c96.
2024-11-17T20:36:50,746 INFO  [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 30bad7b694a399a1221a097a4c1c0c96 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-17T20:36:50,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/30bad7b694a399a1221a097a4c1c0c96/.tmp/info/5cf54840576e470aac518289e9b765b3 is 1080, key is row0001/info:/1731875810569/Put/seqid=0
2024-11-17T20:36:50,770 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34555 is added to blk_1073741837_1013 (size=6033)
2024-11-17T20:36:50,771 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37709 is added to blk_1073741837_1013 (size=6033)
2024-11-17T20:36:50,771 INFO  [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/30bad7b694a399a1221a097a4c1c0c96/.tmp/info/5cf54840576e470aac518289e9b765b3
2024-11-17T20:36:50,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/30bad7b694a399a1221a097a4c1c0c96/.tmp/info/5cf54840576e470aac518289e9b765b3 as hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/30bad7b694a399a1221a097a4c1c0c96/info/5cf54840576e470aac518289e9b765b3
2024-11-17T20:36:50,785 INFO  [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/30bad7b694a399a1221a097a4c1c0c96/info/5cf54840576e470aac518289e9b765b3, entries=1, sequenceid=5, filesize=5.9 K
2024-11-17T20:36:50,786 INFO  [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 30bad7b694a399a1221a097a4c1c0c96 in 40ms, sequenceid=5, compaction requested=false
2024-11-17T20:36:50,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 30bad7b694a399a1221a097a4c1c0c96:

2024-11-17T20:36:50,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731875800506.30bad7b694a399a1221a097a4c1c0c96.
2024-11-17T20:36:50,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8
2024-11-17T20:36:50,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36055 {}] master.HMaster(4169): Remote procedure done, pid=8
2024-11-17T20:36:50,794 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7
2024-11-17T20:36:50,794 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 207 msec
2024-11-17T20:36:50,796 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 218 msec
2024-11-17T20:36:50,796 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:50,809 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:50,968 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:51,796 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:51,810 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:51,969 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:52,797 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:52,811 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:52,969 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:53,797 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:53,811 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:53,970 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:54,798 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:54,812 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:54,971 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:55,799 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:55,799 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295 after 68055ms
java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor204.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:36:55,812 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:55,813 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta after 68044ms
java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor204.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T20:36:55,971 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:56,800 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:56,813 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:56,972 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:57,801 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:57,814 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:57,973 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:58,801 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:58,814 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:58,974 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:59,802 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:59,815 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:36:59,974 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:00,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7
2024-11-17T20:37:00,589 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-11-17T20:37:00,595 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36055 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-17T20:37:00,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36055 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-17T20:37:00,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9
2024-11-17T20:37:00,600 INFO  [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-11-17T20:37:00,603 INFO  [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-11-17T20:37:00,603 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-11-17T20:37:00,758 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38743 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10
2024-11-17T20:37:00,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731875800506.30bad7b694a399a1221a097a4c1c0c96.
2024-11-17T20:37:00,758 INFO  [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 30bad7b694a399a1221a097a4c1c0c96 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-17T20:37:00,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/30bad7b694a399a1221a097a4c1c0c96/.tmp/info/cf43c6188d3e45f3890aa69c6cf4f7eb is 1080, key is row0002/info:/1731875820591/Put/seqid=0
2024-11-17T20:37:00,769 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34555 is added to blk_1073741838_1014 (size=6033)
2024-11-17T20:37:00,770 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37709 is added to blk_1073741838_1014 (size=6033)
2024-11-17T20:37:00,771 INFO  [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/30bad7b694a399a1221a097a4c1c0c96/.tmp/info/cf43c6188d3e45f3890aa69c6cf4f7eb
2024-11-17T20:37:00,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/30bad7b694a399a1221a097a4c1c0c96/.tmp/info/cf43c6188d3e45f3890aa69c6cf4f7eb as hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/30bad7b694a399a1221a097a4c1c0c96/info/cf43c6188d3e45f3890aa69c6cf4f7eb
2024-11-17T20:37:00,784 INFO  [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/30bad7b694a399a1221a097a4c1c0c96/info/cf43c6188d3e45f3890aa69c6cf4f7eb, entries=1, sequenceid=9, filesize=5.9 K
2024-11-17T20:37:00,785 INFO  [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 30bad7b694a399a1221a097a4c1c0c96 in 27ms, sequenceid=9, compaction requested=false
2024-11-17T20:37:00,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 30bad7b694a399a1221a097a4c1c0c96:

2024-11-17T20:37:00,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731875800506.30bad7b694a399a1221a097a4c1c0c96.
2024-11-17T20:37:00,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10
2024-11-17T20:37:00,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36055 {}] master.HMaster(4169): Remote procedure done, pid=10
2024-11-17T20:37:00,789 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9
2024-11-17T20:37:00,789 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 184 msec
2024-11-17T20:37:00,792 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 195 msec
2024-11-17T20:37:00,803 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:00,815 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:00,975 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:01,803 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:01,816 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:01,976 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:02,804 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:02,816 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:02,977 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:03,804 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:03,817 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:03,977 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:04,805 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:04,818 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:04,978 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:05,806 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:05,818 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:05,978 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:06,806 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:06,819 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:06,979 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:07,807 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:07,819 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:07,980 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:08,807 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:08,820 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:08,980 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:09,104 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-17T20:37:09,808 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:09,820 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:09,981 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:10,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9
2024-11-17T20:37:10,697 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-11-17T20:37:10,700 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0b0b0ae8de03%2C38743%2C1731875799347.1731875830700
2024-11-17T20:37:10,710 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:37:10,710 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:37:10,711 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:37:10,711 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:37:10,711 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:37:10,711 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/WALs/0b0b0ae8de03,38743,1731875799347/0b0b0ae8de03%2C38743%2C1731875799347.1731875799988 with entries=8, filesize=5.41 KB; new WAL /user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/WALs/0b0b0ae8de03,38743,1731875799347/0b0b0ae8de03%2C38743%2C1731875799347.1731875830700
2024-11-17T20:37:10,712 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37705:37705),(127.0.0.1/127.0.0.1:46215:46215)]
2024-11-17T20:37:10,712 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/WALs/0b0b0ae8de03,38743,1731875799347/0b0b0ae8de03%2C38743%2C1731875799347.1731875799988 is not closed yet, will try archiving it next time
2024-11-17T20:37:10,713 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34555 is added to blk_1073741833_1009 (size=5546)
2024-11-17T20:37:10,713 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37709 is added to blk_1073741833_1009 (size=5546)
2024-11-17T20:37:10,713 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36055 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-17T20:37:10,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36055 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-17T20:37:10,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11
2024-11-17T20:37:10,716 INFO  [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-11-17T20:37:10,717 INFO  [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-11-17T20:37:10,717 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-11-17T20:37:10,809 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:10,821 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:10,869 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38743 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12
2024-11-17T20:37:10,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731875800506.30bad7b694a399a1221a097a4c1c0c96.
2024-11-17T20:37:10,870 INFO  [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 30bad7b694a399a1221a097a4c1c0c96 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-17T20:37:10,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/30bad7b694a399a1221a097a4c1c0c96/.tmp/info/e0d1ff0256764d008a4f64c40be27f9a is 1080, key is row0003/info:/1731875830699/Put/seqid=0
2024-11-17T20:37:10,885 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34555 is added to blk_1073741840_1016 (size=6033)
2024-11-17T20:37:10,885 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37709 is added to blk_1073741840_1016 (size=6033)
2024-11-17T20:37:10,886 INFO  [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/30bad7b694a399a1221a097a4c1c0c96/.tmp/info/e0d1ff0256764d008a4f64c40be27f9a
2024-11-17T20:37:10,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/30bad7b694a399a1221a097a4c1c0c96/.tmp/info/e0d1ff0256764d008a4f64c40be27f9a as hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/30bad7b694a399a1221a097a4c1c0c96/info/e0d1ff0256764d008a4f64c40be27f9a
2024-11-17T20:37:10,902 INFO  [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/30bad7b694a399a1221a097a4c1c0c96/info/e0d1ff0256764d008a4f64c40be27f9a, entries=1, sequenceid=13, filesize=5.9 K
2024-11-17T20:37:10,903 INFO  [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 30bad7b694a399a1221a097a4c1c0c96 in 33ms, sequenceid=13, compaction requested=true
2024-11-17T20:37:10,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 30bad7b694a399a1221a097a4c1c0c96:

2024-11-17T20:37:10,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731875800506.30bad7b694a399a1221a097a4c1c0c96.
2024-11-17T20:37:10,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12
2024-11-17T20:37:10,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36055 {}] master.HMaster(4169): Remote procedure done, pid=12
2024-11-17T20:37:10,908 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11
2024-11-17T20:37:10,908 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 188 msec
2024-11-17T20:37:10,910 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 196 msec
2024-11-17T20:37:10,982 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:11,810 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:11,822 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:11,982 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:12,810 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:12,822 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:12,983 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:13,811 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:13,823 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:13,983 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:14,812 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:14,823 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:14,984 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:15,813 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:15,824 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:15,985 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:16,813 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:16,825 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:16,985 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:17,814 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:17,825 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:17,986 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:18,815 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:18,826 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:18,987 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:19,815 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:19,826 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:19,988 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:20,487 INFO  [master/0b0b0ae8de03:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-11-17T20:37:20,487 INFO  [master/0b0b0ae8de03:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-11-17T20:37:20,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11
2024-11-17T20:37:20,737 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-11-17T20:37:20,737 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-17T20:37:20,739 DEBUG [Time-limited test {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 18099 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-17T20:37:20,739 DEBUG [Time-limited test {}] regionserver.HStore(1541): 30bad7b694a399a1221a097a4c1c0c96/info is initiating minor compaction (all files)
2024-11-17T20:37:20,739 INFO  [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-17T20:37:20,739 INFO  [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-17T20:37:20,739 INFO  [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 30bad7b694a399a1221a097a4c1c0c96/info in TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731875800506.30bad7b694a399a1221a097a4c1c0c96.
2024-11-17T20:37:20,739 INFO  [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/30bad7b694a399a1221a097a4c1c0c96/info/5cf54840576e470aac518289e9b765b3, hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/30bad7b694a399a1221a097a4c1c0c96/info/cf43c6188d3e45f3890aa69c6cf4f7eb, hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/30bad7b694a399a1221a097a4c1c0c96/info/e0d1ff0256764d008a4f64c40be27f9a] into tmpdir=hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/30bad7b694a399a1221a097a4c1c0c96/.tmp, totalSize=17.7 K
2024-11-17T20:37:20,740 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 5cf54840576e470aac518289e9b765b3, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1731875810569
2024-11-17T20:37:20,740 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting cf43c6188d3e45f3890aa69c6cf4f7eb, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=9, earliestPutTs=1731875820591
2024-11-17T20:37:20,741 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting e0d1ff0256764d008a4f64c40be27f9a, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731875830699
2024-11-17T20:37:20,755 INFO  [Time-limited test {}] throttle.PressureAwareThroughputController(145): 30bad7b694a399a1221a097a4c1c0c96#info#compaction#45 average throughput is 3.08 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-17T20:37:20,755 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/30bad7b694a399a1221a097a4c1c0c96/.tmp/info/6b447c9144344143b8cd36d61e31d5a6 is 1080, key is row0001/info:/1731875810569/Put/seqid=0
2024-11-17T20:37:20,760 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37709 is added to blk_1073741841_1017 (size=8296)
2024-11-17T20:37:20,761 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34555 is added to blk_1073741841_1017 (size=8296)
2024-11-17T20:37:20,767 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/30bad7b694a399a1221a097a4c1c0c96/.tmp/info/6b447c9144344143b8cd36d61e31d5a6 as hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/30bad7b694a399a1221a097a4c1c0c96/info/6b447c9144344143b8cd36d61e31d5a6
2024-11-17T20:37:20,774 INFO  [Time-limited test {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 30bad7b694a399a1221a097a4c1c0c96/info of 30bad7b694a399a1221a097a4c1c0c96 into 6b447c9144344143b8cd36d61e31d5a6(size=8.1 K), total size for store is 8.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-17T20:37:20,775 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 30bad7b694a399a1221a097a4c1c0c96:

2024-11-17T20:37:20,777 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0b0b0ae8de03%2C38743%2C1731875799347.1731875840777
2024-11-17T20:37:20,783 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:37:20,783 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:37:20,783 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:37:20,783 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:37:20,783 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:37:20,784 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/WALs/0b0b0ae8de03,38743,1731875799347/0b0b0ae8de03%2C38743%2C1731875799347.1731875830700 with entries=4, filesize=2.45 KB; new WAL /user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/WALs/0b0b0ae8de03,38743,1731875799347/0b0b0ae8de03%2C38743%2C1731875799347.1731875840777
2024-11-17T20:37:20,784 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46215:46215),(127.0.0.1/127.0.0.1:37705:37705)]
2024-11-17T20:37:20,784 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/WALs/0b0b0ae8de03,38743,1731875799347/0b0b0ae8de03%2C38743%2C1731875799347.1731875830700 is not closed yet, will try archiving it next time
2024-11-17T20:37:20,785 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34555 is added to blk_1073741839_1015 (size=2520)
2024-11-17T20:37:20,785 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37709 is added to blk_1073741839_1015 (size=2520)
2024-11-17T20:37:20,787 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36055 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-17T20:37:20,788 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/WALs/0b0b0ae8de03,38743,1731875799347/0b0b0ae8de03%2C38743%2C1731875799347.1731875799988 to hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/oldWALs/0b0b0ae8de03%2C38743%2C1731875799347.1731875799988
2024-11-17T20:37:20,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36055 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-17T20:37:20,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13
2024-11-17T20:37:20,790 INFO  [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-11-17T20:37:20,791 INFO  [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-11-17T20:37:20,791 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-11-17T20:37:20,816 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:20,827 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:20,944 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38743 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14
2024-11-17T20:37:20,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731875800506.30bad7b694a399a1221a097a4c1c0c96.
2024-11-17T20:37:20,944 INFO  [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 30bad7b694a399a1221a097a4c1c0c96 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-17T20:37:20,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/30bad7b694a399a1221a097a4c1c0c96/.tmp/info/51078e26147844748231c055ca9e9e3b is 1080, key is row0000/info:/1731875840776/Put/seqid=0
2024-11-17T20:37:20,956 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34555 is added to blk_1073741843_1019 (size=6033)
2024-11-17T20:37:20,956 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37709 is added to blk_1073741843_1019 (size=6033)
2024-11-17T20:37:20,957 INFO  [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/30bad7b694a399a1221a097a4c1c0c96/.tmp/info/51078e26147844748231c055ca9e9e3b
2024-11-17T20:37:20,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/30bad7b694a399a1221a097a4c1c0c96/.tmp/info/51078e26147844748231c055ca9e9e3b as hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/30bad7b694a399a1221a097a4c1c0c96/info/51078e26147844748231c055ca9e9e3b
2024-11-17T20:37:20,971 INFO  [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/30bad7b694a399a1221a097a4c1c0c96/info/51078e26147844748231c055ca9e9e3b, entries=1, sequenceid=18, filesize=5.9 K
2024-11-17T20:37:20,972 INFO  [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 30bad7b694a399a1221a097a4c1c0c96 in 28ms, sequenceid=18, compaction requested=false
2024-11-17T20:37:20,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 30bad7b694a399a1221a097a4c1c0c96:

2024-11-17T20:37:20,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731875800506.30bad7b694a399a1221a097a4c1c0c96.
2024-11-17T20:37:20,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14
2024-11-17T20:37:20,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36055 {}] master.HMaster(4169): Remote procedure done, pid=14
2024-11-17T20:37:20,976 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13
2024-11-17T20:37:20,976 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 182 msec
2024-11-17T20:37:20,978 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 189 msec
2024-11-17T20:37:20,988 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:21,186 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/WALs/0b0b0ae8de03,38743,1731875799347/0b0b0ae8de03%2C38743%2C1731875799347.1731875830700 to hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/oldWALs/0b0b0ae8de03%2C38743%2C1731875799347.1731875830700
2024-11-17T20:37:21,817 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:21,827 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:21,989 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:22,817 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:22,827 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:22,990 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:23,818 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:23,828 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:23,990 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:24,819 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:24,828 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:24,991 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:25,819 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:25,829 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:25,859 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 30bad7b694a399a1221a097a4c1c0c96, had cached 0 bytes from a total of 14329
2024-11-17T20:37:25,992 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:26,820 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:26,829 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:26,992 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:27,820 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:27,830 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:27,993 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:28,821 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:28,830 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:28,994 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:29,822 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:29,831 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:29,994 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:30,822 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:30,832 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:30,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13
2024-11-17T20:37:30,898 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-11-17T20:37:30,901 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0b0b0ae8de03%2C38743%2C1731875799347.1731875850901
2024-11-17T20:37:30,909 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:37:30,909 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:37:30,909 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:37:30,909 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:37:30,910 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:37:30,910 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/WALs/0b0b0ae8de03,38743,1731875799347/0b0b0ae8de03%2C38743%2C1731875799347.1731875840777 with entries=3, filesize=1.97 KB; new WAL /user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/WALs/0b0b0ae8de03,38743,1731875799347/0b0b0ae8de03%2C38743%2C1731875799347.1731875850901
2024-11-17T20:37:30,911 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37705:37705),(127.0.0.1/127.0.0.1:46215:46215)]
2024-11-17T20:37:30,911 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/WALs/0b0b0ae8de03,38743,1731875799347/0b0b0ae8de03%2C38743%2C1731875799347.1731875840777 is not closed yet, will try archiving it next time
2024-11-17T20:37:30,911 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-17T20:37:30,911 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-17T20:37:30,912 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-17T20:37:30,912 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T20:37:30,912 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T20:37:30,912 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-17T20:37:30,912 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37709 is added to blk_1073741842_1018 (size=2026)
2024-11-17T20:37:30,912 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34555 is added to blk_1073741842_1018 (size=2026)
2024-11-17T20:37:30,913 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-17T20:37:30,913 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=999169588, stopped=false
2024-11-17T20:37:30,913 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=0b0b0ae8de03,36055,1731875799124
2024-11-17T20:37:30,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36055-0x1014a837bee0000, quorum=127.0.0.1:50742, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-17T20:37:30,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38743-0x1014a837bee0001, quorum=127.0.0.1:50742, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-17T20:37:30,956 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-17T20:37:30,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36055-0x1014a837bee0000, quorum=127.0.0.1:50742, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:37:30,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38743-0x1014a837bee0001, quorum=127.0.0.1:50742, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:37:30,957 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-17T20:37:30,957 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-17T20:37:30,958 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T20:37:30,958 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36055-0x1014a837bee0000, quorum=127.0.0.1:50742, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-17T20:37:30,958 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38743-0x1014a837bee0001, quorum=127.0.0.1:50742, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-17T20:37:30,958 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '0b0b0ae8de03,38743,1731875799347' *****
2024-11-17T20:37:30,958 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-17T20:37:30,959 INFO  [RS:0;0b0b0ae8de03:38743 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-17T20:37:30,959 INFO  [RS:0;0b0b0ae8de03:38743 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-17T20:37:30,959 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-17T20:37:30,959 INFO  [RS:0;0b0b0ae8de03:38743 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-17T20:37:30,960 INFO  [RS:0;0b0b0ae8de03:38743 {}] regionserver.HRegionServer(3091): Received CLOSE for 30bad7b694a399a1221a097a4c1c0c96
2024-11-17T20:37:30,960 INFO  [RS:0;0b0b0ae8de03:38743 {}] regionserver.HRegionServer(959): stopping server 0b0b0ae8de03,38743,1731875799347
2024-11-17T20:37:30,960 INFO  [RS:0;0b0b0ae8de03:38743 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-17T20:37:30,960 INFO  [RS:0;0b0b0ae8de03:38743 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;0b0b0ae8de03:38743.
2024-11-17T20:37:30,960 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 30bad7b694a399a1221a097a4c1c0c96, disabling compactions & flushes
2024-11-17T20:37:30,960 INFO  [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731875800506.30bad7b694a399a1221a097a4c1c0c96.
2024-11-17T20:37:30,960 DEBUG [RS:0;0b0b0ae8de03:38743 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-17T20:37:30,960 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731875800506.30bad7b694a399a1221a097a4c1c0c96.
2024-11-17T20:37:30,961 DEBUG [RS:0;0b0b0ae8de03:38743 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T20:37:30,961 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731875800506.30bad7b694a399a1221a097a4c1c0c96. after waiting 0 ms
2024-11-17T20:37:30,961 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731875800506.30bad7b694a399a1221a097a4c1c0c96.
2024-11-17T20:37:30,961 INFO  [RS:0;0b0b0ae8de03:38743 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-17T20:37:30,961 INFO  [RS:0;0b0b0ae8de03:38743 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-17T20:37:30,961 INFO  [RS:0;0b0b0ae8de03:38743 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-17T20:37:30,961 INFO  [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 30bad7b694a399a1221a097a4c1c0c96 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-17T20:37:30,961 INFO  [RS:0;0b0b0ae8de03:38743 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-17T20:37:30,961 INFO  [RS:0;0b0b0ae8de03:38743 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-11-17T20:37:30,961 DEBUG [RS:0;0b0b0ae8de03:38743 {}] regionserver.HRegionServer(1325): Online Regions={30bad7b694a399a1221a097a4c1c0c96=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731875800506.30bad7b694a399a1221a097a4c1c0c96., 1588230740=hbase:meta,,1.1588230740}
2024-11-17T20:37:30,962 DEBUG [RS:0;0b0b0ae8de03:38743 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 30bad7b694a399a1221a097a4c1c0c96
2024-11-17T20:37:30,962 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-17T20:37:30,962 INFO  [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-17T20:37:30,962 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-17T20:37:30,962 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-17T20:37:30,962 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-17T20:37:30,962 INFO  [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.89 KB heapSize=3.91 KB
2024-11-17T20:37:30,966 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/30bad7b694a399a1221a097a4c1c0c96/.tmp/info/1214017b3c024997adc56be645210f3c is 1080, key is row0001/info:/1731875850899/Put/seqid=0
2024-11-17T20:37:30,971 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37709 is added to blk_1073741845_1021 (size=6033)
2024-11-17T20:37:30,971 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34555 is added to blk_1073741845_1021 (size=6033)
2024-11-17T20:37:30,971 INFO  [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=22 (bloomFilter=true), to=hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/30bad7b694a399a1221a097a4c1c0c96/.tmp/info/1214017b3c024997adc56be645210f3c
2024-11-17T20:37:30,977 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/30bad7b694a399a1221a097a4c1c0c96/.tmp/info/1214017b3c024997adc56be645210f3c as hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/30bad7b694a399a1221a097a4c1c0c96/info/1214017b3c024997adc56be645210f3c
2024-11-17T20:37:30,981 INFO  [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/30bad7b694a399a1221a097a4c1c0c96/info/1214017b3c024997adc56be645210f3c, entries=1, sequenceid=22, filesize=5.9 K
2024-11-17T20:37:30,982 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/data/hbase/meta/1588230740/.tmp/info/9ea269932b3549dda7f9f3d203602619 is 227, key is TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731875800506.30bad7b694a399a1221a097a4c1c0c96./info:regioninfo/1731875800875/Put/seqid=0
2024-11-17T20:37:30,983 INFO  [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 30bad7b694a399a1221a097a4c1c0c96 in 21ms, sequenceid=22, compaction requested=true
2024-11-17T20:37:30,983 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731875800506.30bad7b694a399a1221a097a4c1c0c96.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/30bad7b694a399a1221a097a4c1c0c96/info/5cf54840576e470aac518289e9b765b3, hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/30bad7b694a399a1221a097a4c1c0c96/info/cf43c6188d3e45f3890aa69c6cf4f7eb, hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/30bad7b694a399a1221a097a4c1c0c96/info/e0d1ff0256764d008a4f64c40be27f9a] to archive
2024-11-17T20:37:30,984 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731875800506.30bad7b694a399a1221a097a4c1c0c96.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-17T20:37:30,986 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731875800506.30bad7b694a399a1221a097a4c1c0c96.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/30bad7b694a399a1221a097a4c1c0c96/info/5cf54840576e470aac518289e9b765b3 to hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/30bad7b694a399a1221a097a4c1c0c96/info/5cf54840576e470aac518289e9b765b3
2024-11-17T20:37:30,986 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37709 is added to blk_1073741846_1022 (size=7308)
2024-11-17T20:37:30,987 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34555 is added to blk_1073741846_1022 (size=7308)
2024-11-17T20:37:30,987 INFO  [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/data/hbase/meta/1588230740/.tmp/info/9ea269932b3549dda7f9f3d203602619
2024-11-17T20:37:30,987 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731875800506.30bad7b694a399a1221a097a4c1c0c96.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/30bad7b694a399a1221a097a4c1c0c96/info/cf43c6188d3e45f3890aa69c6cf4f7eb to hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/30bad7b694a399a1221a097a4c1c0c96/info/cf43c6188d3e45f3890aa69c6cf4f7eb
2024-11-17T20:37:30,988 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731875800506.30bad7b694a399a1221a097a4c1c0c96.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/30bad7b694a399a1221a097a4c1c0c96/info/e0d1ff0256764d008a4f64c40be27f9a to hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/30bad7b694a399a1221a097a4c1c0c96/info/e0d1ff0256764d008a4f64c40be27f9a
2024-11-17T20:37:30,988 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731875800506.30bad7b694a399a1221a097a4c1c0c96.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=0b0b0ae8de03:36055 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 16 more
2024-11-17T20:37:30,989 WARN  [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731875800506.30bad7b694a399a1221a097a4c1c0c96.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [5cf54840576e470aac518289e9b765b3=6033, cf43c6188d3e45f3890aa69c6cf4f7eb=6033, e0d1ff0256764d008a4f64c40be27f9a=6033]
2024-11-17T20:37:30,992 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/30bad7b694a399a1221a097a4c1c0c96/recovered.edits/25.seqid, newMaxSeqId=25, maxSeqId=1
2024-11-17T20:37:30,993 INFO  [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731875800506.30bad7b694a399a1221a097a4c1c0c96.
2024-11-17T20:37:30,993 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 30bad7b694a399a1221a097a4c1c0c96:
Waiting for close lock at 1731875850960Running coprocessor pre-close hooks at 1731875850960Disabling compacts and flushes for region at 1731875850960Disabling writes for close at 1731875850961 (+1 ms)Obtaining lock to block concurrent updates at 1731875850961Preparing flush snapshotting stores in 30bad7b694a399a1221a097a4c1c0c96 at 1731875850961Finished memstore snapshotting TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731875800506.30bad7b694a399a1221a097a4c1c0c96., syncing WAL and waiting on mvcc, flushsize=dataSize=1076, getHeapSize=1392, getOffHeapSize=0, getCellsCount=1 at 1731875850961Flushing stores of TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731875800506.30bad7b694a399a1221a097a4c1c0c96. at 1731875850962 (+1 ms)Flushing 30bad7b694a399a1221a097a4c1c0c96/info: creating writer at 1731875850963 (+1 ms)Flushing 30bad7b694a399a1221a097a4c1c0c96/info: appending metadata at 1731875850966 (+3 ms)Flushing 30bad7b694a399a1221a097a4c1c0c96/info: closing flushed file at 1731875850966Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3b0fd4eb: reopening flushed file at 1731875850976 (+10 ms)Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 30bad7b694a399a1221a097a4c1c0c96 in 21ms, sequenceid=22, compaction requested=true at 1731875850983 (+7 ms)Writing region close event to WAL at 1731875850989 (+6 ms)Running coprocessor post-close hooks at 1731875850993 (+4 ms)Closed at 1731875850993
2024-11-17T20:37:30,993 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731875800506.30bad7b694a399a1221a097a4c1c0c96.
2024-11-17T20:37:30,995 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:31,007 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/data/hbase/meta/1588230740/.tmp/ns/2f066ca2b6da4552a5593d1887bb90be is 43, key is default/ns:d/1731875800421/Put/seqid=0
2024-11-17T20:37:31,011 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34555 is added to blk_1073741847_1023 (size=5153)
2024-11-17T20:37:31,011 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37709 is added to blk_1073741847_1023 (size=5153)
2024-11-17T20:37:31,012 INFO  [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/data/hbase/meta/1588230740/.tmp/ns/2f066ca2b6da4552a5593d1887bb90be
2024-11-17T20:37:31,035 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/data/hbase/meta/1588230740/.tmp/table/76dda5fbab694e05a3518a12318eb324 is 89, key is TestLogRolling-testCompactionRecordDoesntBlockRolling/table:state/1731875800885/Put/seqid=0
2024-11-17T20:37:31,040 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37709 is added to blk_1073741848_1024 (size=5508)
2024-11-17T20:37:31,040 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34555 is added to blk_1073741848_1024 (size=5508)
2024-11-17T20:37:31,041 INFO  [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=170 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/data/hbase/meta/1588230740/.tmp/table/76dda5fbab694e05a3518a12318eb324
2024-11-17T20:37:31,047 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/data/hbase/meta/1588230740/.tmp/info/9ea269932b3549dda7f9f3d203602619 as hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/data/hbase/meta/1588230740/info/9ea269932b3549dda7f9f3d203602619
2024-11-17T20:37:31,053 INFO  [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/data/hbase/meta/1588230740/info/9ea269932b3549dda7f9f3d203602619, entries=10, sequenceid=11, filesize=7.1 K
2024-11-17T20:37:31,054 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/data/hbase/meta/1588230740/.tmp/ns/2f066ca2b6da4552a5593d1887bb90be as hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/data/hbase/meta/1588230740/ns/2f066ca2b6da4552a5593d1887bb90be
2024-11-17T20:37:31,059 INFO  [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/data/hbase/meta/1588230740/ns/2f066ca2b6da4552a5593d1887bb90be, entries=2, sequenceid=11, filesize=5.0 K
2024-11-17T20:37:31,060 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/data/hbase/meta/1588230740/.tmp/table/76dda5fbab694e05a3518a12318eb324 as hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/data/hbase/meta/1588230740/table/76dda5fbab694e05a3518a12318eb324
2024-11-17T20:37:31,065 INFO  [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/data/hbase/meta/1588230740/table/76dda5fbab694e05a3518a12318eb324, entries=2, sequenceid=11, filesize=5.4 K
2024-11-17T20:37:31,066 INFO  [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.89 KB/1932, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1588230740 in 104ms, sequenceid=11, compaction requested=false
2024-11-17T20:37:31,070 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-11-17T20:37:31,070 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-17T20:37:31,070 INFO  [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-17T20:37:31,070 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731875850962Running coprocessor pre-close hooks at 1731875850962Disabling compacts and flushes for region at 1731875850962Disabling writes for close at 1731875850962Obtaining lock to block concurrent updates at 1731875850962Preparing flush snapshotting stores in 1588230740 at 1731875850962Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1932, getHeapSize=3936, getOffHeapSize=0, getCellsCount=14 at 1731875850963 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731875850963Flushing 1588230740/info: creating writer at 1731875850963Flushing 1588230740/info: appending metadata at 1731875850981 (+18 ms)Flushing 1588230740/info: closing flushed file at 1731875850981Flushing 1588230740/ns: creating writer at 1731875850992 (+11 ms)Flushing 1588230740/ns: appending metadata at 1731875851006 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1731875851007 (+1 ms)Flushing 1588230740/table: creating writer at 1731875851016 (+9 ms)Flushing 1588230740/table: appending metadata at 1731875851035 (+19 ms)Flushing 1588230740/table: closing flushed file at 1731875851035Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6639b77c: reopening flushed file at 1731875851046 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@276e36bc: reopening flushed file at 1731875851053 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7508e5dc: reopening flushed file at 1731875851059 (+6 ms)Finished flush of dataSize ~1.89 KB/1932, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1588230740 in 104ms, sequenceid=11, compaction requested=false at 1731875851066 (+7 ms)Writing region close event to WAL at 1731875851067 (+1 ms)Running coprocessor post-close hooks at 1731875851070 (+3 ms)Closed at 1731875851070
2024-11-17T20:37:31,070 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-17T20:37:31,162 INFO  [RS:0;0b0b0ae8de03:38743 {}] regionserver.HRegionServer(976): stopping server 0b0b0ae8de03,38743,1731875799347; all regions closed.
2024-11-17T20:37:31,162 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:37:31,163 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:37:31,163 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:37:31,163 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:37:31,163 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:37:31,166 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34555 is added to blk_1073741834_1010 (size=3306)
2024-11-17T20:37:31,166 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37709 is added to blk_1073741834_1010 (size=3306)
2024-11-17T20:37:31,170 DEBUG [RS:0;0b0b0ae8de03:38743 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/oldWALs
2024-11-17T20:37:31,170 INFO  [RS:0;0b0b0ae8de03:38743 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 0b0b0ae8de03%2C38743%2C1731875799347.meta:.meta(num 1731875800339)
2024-11-17T20:37:31,170 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:37:31,171 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:37:31,171 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:37:31,171 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:37:31,171 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:37:31,173 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34555 is added to blk_1073741844_1020 (size=1252)
2024-11-17T20:37:31,174 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37709 is added to blk_1073741844_1020 (size=1252)
2024-11-17T20:37:31,178 DEBUG [RS:0;0b0b0ae8de03:38743 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/oldWALs
2024-11-17T20:37:31,178 INFO  [RS:0;0b0b0ae8de03:38743 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 0b0b0ae8de03%2C38743%2C1731875799347:(num 1731875850901)
2024-11-17T20:37:31,178 DEBUG [RS:0;0b0b0ae8de03:38743 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T20:37:31,178 INFO  [RS:0;0b0b0ae8de03:38743 {}] regionserver.LeaseManager(133): Closed leases
2024-11-17T20:37:31,178 INFO  [RS:0;0b0b0ae8de03:38743 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-17T20:37:31,178 INFO  [RS:0;0b0b0ae8de03:38743 {}] hbase.ChoreService(370): Chore service for: regionserver/0b0b0ae8de03:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-17T20:37:31,178 INFO  [RS:0;0b0b0ae8de03:38743 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-17T20:37:31,179 INFO  [regionserver/0b0b0ae8de03:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-17T20:37:31,179 INFO  [RS:0;0b0b0ae8de03:38743 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38743
2024-11-17T20:37:31,187 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38743-0x1014a837bee0001, quorum=127.0.0.1:50742, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/0b0b0ae8de03,38743,1731875799347
2024-11-17T20:37:31,187 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36055-0x1014a837bee0000, quorum=127.0.0.1:50742, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-17T20:37:31,187 INFO  [RS:0;0b0b0ae8de03:38743 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-17T20:37:31,198 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [0b0b0ae8de03,38743,1731875799347]
2024-11-17T20:37:31,209 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/0b0b0ae8de03,38743,1731875799347 already deleted, retry=false
2024-11-17T20:37:31,209 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 0b0b0ae8de03,38743,1731875799347 expired; onlineServers=0
2024-11-17T20:37:31,209 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '0b0b0ae8de03,36055,1731875799124' *****
2024-11-17T20:37:31,209 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-17T20:37:31,209 INFO  [M:0;0b0b0ae8de03:36055 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-17T20:37:31,209 INFO  [M:0;0b0b0ae8de03:36055 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-17T20:37:31,209 DEBUG [M:0;0b0b0ae8de03:36055 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-17T20:37:31,209 DEBUG [M:0;0b0b0ae8de03:36055 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-17T20:37:31,209 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-17T20:37:31,210 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster-HFileCleaner.large.0-1731875799707 {}] cleaner.HFileCleaner(306): Exit Thread[master/0b0b0ae8de03:0:becomeActiveMaster-HFileCleaner.large.0-1731875799707,5,FailOnTimeoutGroup]
2024-11-17T20:37:31,210 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster-HFileCleaner.small.0-1731875799707 {}] cleaner.HFileCleaner(306): Exit Thread[master/0b0b0ae8de03:0:becomeActiveMaster-HFileCleaner.small.0-1731875799707,5,FailOnTimeoutGroup]
2024-11-17T20:37:31,210 INFO  [M:0;0b0b0ae8de03:36055 {}] hbase.ChoreService(370): Chore service for: master/0b0b0ae8de03:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-17T20:37:31,210 INFO  [M:0;0b0b0ae8de03:36055 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-17T20:37:31,210 DEBUG [M:0;0b0b0ae8de03:36055 {}] master.HMaster(1795): Stopping service threads
2024-11-17T20:37:31,210 INFO  [M:0;0b0b0ae8de03:36055 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-17T20:37:31,210 INFO  [M:0;0b0b0ae8de03:36055 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-17T20:37:31,211 INFO  [M:0;0b0b0ae8de03:36055 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-17T20:37:31,211 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-17T20:37:31,219 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36055-0x1014a837bee0000, quorum=127.0.0.1:50742, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-17T20:37:31,219 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36055-0x1014a837bee0000, quorum=127.0.0.1:50742, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:37:31,219 DEBUG [M:0;0b0b0ae8de03:36055 {}] zookeeper.ZKUtil(347): master:36055-0x1014a837bee0000, quorum=127.0.0.1:50742, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-17T20:37:31,219 WARN  [M:0;0b0b0ae8de03:36055 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-17T20:37:31,220 INFO  [M:0;0b0b0ae8de03:36055 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/.lastflushedseqids
2024-11-17T20:37:31,225 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34555 is added to blk_1073741849_1025 (size=130)
2024-11-17T20:37:31,225 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37709 is added to blk_1073741849_1025 (size=130)
2024-11-17T20:37:31,225 INFO  [M:0;0b0b0ae8de03:36055 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-17T20:37:31,226 INFO  [M:0;0b0b0ae8de03:36055 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-17T20:37:31,226 DEBUG [M:0;0b0b0ae8de03:36055 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-17T20:37:31,226 INFO  [M:0;0b0b0ae8de03:36055 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T20:37:31,226 DEBUG [M:0;0b0b0ae8de03:36055 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T20:37:31,226 DEBUG [M:0;0b0b0ae8de03:36055 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-17T20:37:31,226 DEBUG [M:0;0b0b0ae8de03:36055 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T20:37:31,226 INFO  [M:0;0b0b0ae8de03:36055 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=43.56 KB heapSize=54.94 KB
2024-11-17T20:37:31,239 DEBUG [M:0;0b0b0ae8de03:36055 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a98f4f41d95b45449ec8853bf00a09a6 is 82, key is hbase:meta,,1/info:regioninfo/1731875800370/Put/seqid=0
2024-11-17T20:37:31,244 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34555 is added to blk_1073741850_1026 (size=5672)
2024-11-17T20:37:31,244 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37709 is added to blk_1073741850_1026 (size=5672)
2024-11-17T20:37:31,245 INFO  [M:0;0b0b0ae8de03:36055 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a98f4f41d95b45449ec8853bf00a09a6
2024-11-17T20:37:31,263 DEBUG [M:0;0b0b0ae8de03:36055 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c04087e1cbeb4a9b9b1e632ffb6c2b70 is 798, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731875800889/Put/seqid=0
2024-11-17T20:37:31,268 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34555 is added to blk_1073741851_1027 (size=7819)
2024-11-17T20:37:31,268 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37709 is added to blk_1073741851_1027 (size=7819)
2024-11-17T20:37:31,268 INFO  [M:0;0b0b0ae8de03:36055 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.95 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c04087e1cbeb4a9b9b1e632ffb6c2b70
2024-11-17T20:37:31,273 INFO  [M:0;0b0b0ae8de03:36055 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c04087e1cbeb4a9b9b1e632ffb6c2b70
2024-11-17T20:37:31,294 DEBUG [M:0;0b0b0ae8de03:36055 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e130579babfc4af59d8dcb6f54754676 is 69, key is 0b0b0ae8de03,38743,1731875799347/rs:state/1731875799829/Put/seqid=0
2024-11-17T20:37:31,298 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37709 is added to blk_1073741852_1028 (size=5156)
2024-11-17T20:37:31,298 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34555 is added to blk_1073741852_1028 (size=5156)
2024-11-17T20:37:31,298 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38743-0x1014a837bee0001, quorum=127.0.0.1:50742, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-17T20:37:31,298 INFO  [RS:0;0b0b0ae8de03:38743 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-17T20:37:31,298 INFO  [RS:0;0b0b0ae8de03:38743 {}] regionserver.HRegionServer(1031): Exiting; stopping=0b0b0ae8de03,38743,1731875799347; zookeeper connection closed.
2024-11-17T20:37:31,298 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38743-0x1014a837bee0001, quorum=127.0.0.1:50742, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-17T20:37:31,299 INFO  [M:0;0b0b0ae8de03:36055 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e130579babfc4af59d8dcb6f54754676
2024-11-17T20:37:31,299 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@39a92ea1 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@39a92ea1
2024-11-17T20:37:31,299 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-17T20:37:31,317 DEBUG [M:0;0b0b0ae8de03:36055 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/ca3e7c7f48604de681cea9d8018c2741 is 52, key is load_balancer_on/state:d/1731875800502/Put/seqid=0
2024-11-17T20:37:31,321 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34555 is added to blk_1073741853_1029 (size=5056)
2024-11-17T20:37:31,321 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37709 is added to blk_1073741853_1029 (size=5056)
2024-11-17T20:37:31,322 INFO  [M:0;0b0b0ae8de03:36055 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/ca3e7c7f48604de681cea9d8018c2741
2024-11-17T20:37:31,327 DEBUG [M:0;0b0b0ae8de03:36055 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a98f4f41d95b45449ec8853bf00a09a6 as hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a98f4f41d95b45449ec8853bf00a09a6
2024-11-17T20:37:31,332 INFO  [M:0;0b0b0ae8de03:36055 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a98f4f41d95b45449ec8853bf00a09a6, entries=8, sequenceid=121, filesize=5.5 K
2024-11-17T20:37:31,333 DEBUG [M:0;0b0b0ae8de03:36055 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c04087e1cbeb4a9b9b1e632ffb6c2b70 as hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c04087e1cbeb4a9b9b1e632ffb6c2b70
2024-11-17T20:37:31,338 INFO  [M:0;0b0b0ae8de03:36055 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c04087e1cbeb4a9b9b1e632ffb6c2b70
2024-11-17T20:37:31,338 INFO  [M:0;0b0b0ae8de03:36055 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c04087e1cbeb4a9b9b1e632ffb6c2b70, entries=14, sequenceid=121, filesize=7.6 K
2024-11-17T20:37:31,339 DEBUG [M:0;0b0b0ae8de03:36055 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e130579babfc4af59d8dcb6f54754676 as hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e130579babfc4af59d8dcb6f54754676
2024-11-17T20:37:31,343 INFO  [M:0;0b0b0ae8de03:36055 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e130579babfc4af59d8dcb6f54754676, entries=1, sequenceid=121, filesize=5.0 K
2024-11-17T20:37:31,344 DEBUG [M:0;0b0b0ae8de03:36055 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/ca3e7c7f48604de681cea9d8018c2741 as hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/ca3e7c7f48604de681cea9d8018c2741
2024-11-17T20:37:31,349 INFO  [M:0;0b0b0ae8de03:36055 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42353/user/jenkins/test-data/978306a0-0502-9af7-c0ef-ef169ae0225d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/ca3e7c7f48604de681cea9d8018c2741, entries=1, sequenceid=121, filesize=4.9 K
2024-11-17T20:37:31,350 INFO  [M:0;0b0b0ae8de03:36055 {}] regionserver.HRegion(3140): Finished flush of dataSize ~43.56 KB/44602, heapSize ~54.88 KB/56192, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 124ms, sequenceid=121, compaction requested=false
2024-11-17T20:37:31,351 INFO  [M:0;0b0b0ae8de03:36055 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T20:37:31,351 DEBUG [M:0;0b0b0ae8de03:36055 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731875851226Disabling compacts and flushes for region at 1731875851226Disabling writes for close at 1731875851226Obtaining lock to block concurrent updates at 1731875851226Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731875851226Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=44602, getHeapSize=56192, getOffHeapSize=0, getCellsCount=140 at 1731875851226Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731875851227 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731875851227Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731875851239 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731875851239Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731875851249 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731875851262 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731875851262Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731875851274 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731875851293 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731875851293Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731875851303 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731875851317 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731875851317Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@53b80ae0: reopening flushed file at 1731875851326 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@ebed1b9: reopening flushed file at 1731875851332 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2abeff55: reopening flushed file at 1731875851338 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@76b7bdc5: reopening flushed file at 1731875851343 (+5 ms)Finished flush of dataSize ~43.56 KB/44602, heapSize ~54.88 KB/56192, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 124ms, sequenceid=121, compaction requested=false at 1731875851350 (+7 ms)Writing region close event to WAL at 1731875851351 (+1 ms)Closed at 1731875851351
2024-11-17T20:37:31,352 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:37:31,352 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:37:31,352 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:37:31,352 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:37:31,352 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:37:31,354 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34555 is added to blk_1073741830_1006 (size=52999)
2024-11-17T20:37:31,354 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37709 is added to blk_1073741830_1006 (size=52999)
2024-11-17T20:37:31,354 INFO  [M:0;0b0b0ae8de03:36055 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-17T20:37:31,354 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-17T20:37:31,354 INFO  [M:0;0b0b0ae8de03:36055 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36055
2024-11-17T20:37:31,355 INFO  [M:0;0b0b0ae8de03:36055 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-17T20:37:31,467 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36055-0x1014a837bee0000, quorum=127.0.0.1:50742, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-17T20:37:31,467 INFO  [M:0;0b0b0ae8de03:36055 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-17T20:37:31,467 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36055-0x1014a837bee0000, quorum=127.0.0.1:50742, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-17T20:37:31,500 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@c734161{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T20:37:31,500 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@75e789f0{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-17T20:37:31,500 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-17T20:37:31,501 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@14a6d451{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-17T20:37:31,501 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@34534d9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4dd0321-963a-3166-6b1a-243c9e15ca98/hadoop.log.dir/,STOPPED}
2024-11-17T20:37:31,504 WARN  [BP-384780574-172.17.0.2-1731875796225 heartbeating to localhost/127.0.0.1:42353 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-17T20:37:31,504 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-17T20:37:31,504 WARN  [BP-384780574-172.17.0.2-1731875796225 heartbeating to localhost/127.0.0.1:42353 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-384780574-172.17.0.2-1731875796225 (Datanode Uuid 60ab1532-6bfd-42ae-a362-ddccdaa9a30b) service to localhost/127.0.0.1:42353
2024-11-17T20:37:31,504 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-17T20:37:31,505 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4dd0321-963a-3166-6b1a-243c9e15ca98/cluster_3c7e0f58-fc63-0ea1-4594-d31d5a66aa59/data/data3/current/BP-384780574-172.17.0.2-1731875796225 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T20:37:31,505 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4dd0321-963a-3166-6b1a-243c9e15ca98/cluster_3c7e0f58-fc63-0ea1-4594-d31d5a66aa59/data/data4/current/BP-384780574-172.17.0.2-1731875796225 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T20:37:31,505 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-17T20:37:31,507 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@40527499{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T20:37:31,508 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@c1ed8d4{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-17T20:37:31,508 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-17T20:37:31,508 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7c13156e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-17T20:37:31,508 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4faae08f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4dd0321-963a-3166-6b1a-243c9e15ca98/hadoop.log.dir/,STOPPED}
2024-11-17T20:37:31,510 WARN  [BP-384780574-172.17.0.2-1731875796225 heartbeating to localhost/127.0.0.1:42353 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-17T20:37:31,510 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-17T20:37:31,510 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-17T20:37:31,510 WARN  [BP-384780574-172.17.0.2-1731875796225 heartbeating to localhost/127.0.0.1:42353 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-384780574-172.17.0.2-1731875796225 (Datanode Uuid a126db33-8ce9-4169-9e7d-e6315649dbbb) service to localhost/127.0.0.1:42353
2024-11-17T20:37:31,510 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4dd0321-963a-3166-6b1a-243c9e15ca98/cluster_3c7e0f58-fc63-0ea1-4594-d31d5a66aa59/data/data1/current/BP-384780574-172.17.0.2-1731875796225 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T20:37:31,510 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4dd0321-963a-3166-6b1a-243c9e15ca98/cluster_3c7e0f58-fc63-0ea1-4594-d31d5a66aa59/data/data2/current/BP-384780574-172.17.0.2-1731875796225 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T20:37:31,511 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-17T20:37:31,518 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@43498b11{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-17T20:37:31,518 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1367dc96{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-17T20:37:31,518 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-17T20:37:31,518 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@190ad9e7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-17T20:37:31,518 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@20f59884{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4dd0321-963a-3166-6b1a-243c9e15ca98/hadoop.log.dir/,STOPPED}
2024-11-17T20:37:31,523 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-17T20:37:31,542 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-17T20:37:31,550 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testCompactionRecordDoesntBlockRolling Thread=209 (was 183)
Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:42353 from jenkins.hfs.5
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: HMaster-EventLoopGroup-12-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-34-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:42353 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-36-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-37-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-36-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-37-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-12-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:42353 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:42353
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-35-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-35-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: regionserver/0b0b0ae8de03:0.leaseChecker
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.regionserver.LeaseManager.run(LeaseManager.java:82)

Potentially hanging thread: nioEventLoopGroup-34-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-34-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-37-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-35-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-36-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.5@localhost:42353
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:42353
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:42353
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-12-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:42353
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=485 (was 457) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=221 (was 210) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7430 (was 7578)
2024-11-17T20:37:31,559 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRolling Thread=209, OpenFileDescriptor=485, MaxFileDescriptor=1048576, SystemLoadAverage=221, ProcessCount=11, AvailableMemoryMB=7430
2024-11-17T20:37:31,560 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-17T20:37:31,560 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4dd0321-963a-3166-6b1a-243c9e15ca98/hadoop.log.dir so I do NOT create it in target/test-data/1a3196ea-489f-0fa1-8ed8-e2e462983867
2024-11-17T20:37:31,560 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4dd0321-963a-3166-6b1a-243c9e15ca98/hadoop.tmp.dir so I do NOT create it in target/test-data/1a3196ea-489f-0fa1-8ed8-e2e462983867
2024-11-17T20:37:31,560 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1a3196ea-489f-0fa1-8ed8-e2e462983867/cluster_14a128d9-7999-521f-922f-c4fb1ce6abae, deleteOnExit=true
2024-11-17T20:37:31,560 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-17T20:37:31,560 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1a3196ea-489f-0fa1-8ed8-e2e462983867/test.cache.data in system properties and HBase conf
2024-11-17T20:37:31,560 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1a3196ea-489f-0fa1-8ed8-e2e462983867/hadoop.tmp.dir in system properties and HBase conf
2024-11-17T20:37:31,560 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1a3196ea-489f-0fa1-8ed8-e2e462983867/hadoop.log.dir in system properties and HBase conf
2024-11-17T20:37:31,561 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1a3196ea-489f-0fa1-8ed8-e2e462983867/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-17T20:37:31,561 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1a3196ea-489f-0fa1-8ed8-e2e462983867/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-17T20:37:31,561 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-17T20:37:31,561 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-17T20:37:31,561 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1a3196ea-489f-0fa1-8ed8-e2e462983867/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-17T20:37:31,561 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1a3196ea-489f-0fa1-8ed8-e2e462983867/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-17T20:37:31,561 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1a3196ea-489f-0fa1-8ed8-e2e462983867/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-17T20:37:31,561 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1a3196ea-489f-0fa1-8ed8-e2e462983867/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-17T20:37:31,561 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1a3196ea-489f-0fa1-8ed8-e2e462983867/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-17T20:37:31,561 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1a3196ea-489f-0fa1-8ed8-e2e462983867/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-17T20:37:31,561 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1a3196ea-489f-0fa1-8ed8-e2e462983867/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-17T20:37:31,561 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1a3196ea-489f-0fa1-8ed8-e2e462983867/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-17T20:37:31,561 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1a3196ea-489f-0fa1-8ed8-e2e462983867/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-17T20:37:31,562 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1a3196ea-489f-0fa1-8ed8-e2e462983867/nfs.dump.dir in system properties and HBase conf
2024-11-17T20:37:31,562 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1a3196ea-489f-0fa1-8ed8-e2e462983867/java.io.tmpdir in system properties and HBase conf
2024-11-17T20:37:31,562 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1a3196ea-489f-0fa1-8ed8-e2e462983867/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-17T20:37:31,562 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1a3196ea-489f-0fa1-8ed8-e2e462983867/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-17T20:37:31,562 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1a3196ea-489f-0fa1-8ed8-e2e462983867/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-17T20:37:31,577 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-17T20:37:31,823 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:31,832 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:31,854 INFO  [regionserver/0b0b0ae8de03:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-17T20:37:31,926 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-17T20:37:31,929 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-17T20:37:31,930 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-17T20:37:31,930 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-17T20:37:31,930 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-17T20:37:31,931 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-17T20:37:31,931 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@444d0b71{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1a3196ea-489f-0fa1-8ed8-e2e462983867/hadoop.log.dir/,AVAILABLE}
2024-11-17T20:37:31,931 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7193a060{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-17T20:37:31,996 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:32,023 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5fe51576{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1a3196ea-489f-0fa1-8ed8-e2e462983867/java.io.tmpdir/jetty-localhost-38367-hadoop-hdfs-3_4_1-tests_jar-_-any-2995892137956975942/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-17T20:37:32,023 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@284a8a01{HTTP/1.1, (http/1.1)}{localhost:38367}
2024-11-17T20:37:32,023 INFO  [Time-limited test {}] server.Server(415): Started @255309ms
2024-11-17T20:37:32,034 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-17T20:37:32,288 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-17T20:37:32,290 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-17T20:37:32,292 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-17T20:37:32,292 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-17T20:37:32,292 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-17T20:37:32,293 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@601b78f7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1a3196ea-489f-0fa1-8ed8-e2e462983867/hadoop.log.dir/,AVAILABLE}
2024-11-17T20:37:32,293 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@616d506d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-17T20:37:32,386 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@24bba582{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1a3196ea-489f-0fa1-8ed8-e2e462983867/java.io.tmpdir/jetty-localhost-34187-hadoop-hdfs-3_4_1-tests_jar-_-any-16885043397496047281/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T20:37:32,387 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3f5f0b5{HTTP/1.1, (http/1.1)}{localhost:34187}
2024-11-17T20:37:32,387 INFO  [Time-limited test {}] server.Server(415): Started @255673ms
2024-11-17T20:37:32,388 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-17T20:37:32,422 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-17T20:37:32,424 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-17T20:37:32,425 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-17T20:37:32,425 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-17T20:37:32,425 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-17T20:37:32,426 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6f4aa33e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1a3196ea-489f-0fa1-8ed8-e2e462983867/hadoop.log.dir/,AVAILABLE}
2024-11-17T20:37:32,426 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@e483a61{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-17T20:37:32,518 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@42d059d{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1a3196ea-489f-0fa1-8ed8-e2e462983867/java.io.tmpdir/jetty-localhost-37495-hadoop-hdfs-3_4_1-tests_jar-_-any-3803401045841790643/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T20:37:32,519 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@444decb3{HTTP/1.1, (http/1.1)}{localhost:37495}
2024-11-17T20:37:32,519 INFO  [Time-limited test {}] server.Server(415): Started @255805ms
2024-11-17T20:37:32,520 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-17T20:37:32,824 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:32,833 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:32,996 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:33,586 WARN  [Thread-1974 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1a3196ea-489f-0fa1-8ed8-e2e462983867/cluster_14a128d9-7999-521f-922f-c4fb1ce6abae/data/data2/current/BP-909746748-172.17.0.2-1731875851585/current, will proceed with Du for space computation calculation, 
2024-11-17T20:37:33,586 WARN  [Thread-1973 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1a3196ea-489f-0fa1-8ed8-e2e462983867/cluster_14a128d9-7999-521f-922f-c4fb1ce6abae/data/data1/current/BP-909746748-172.17.0.2-1731875851585/current, will proceed with Du for space computation calculation, 
2024-11-17T20:37:33,607 WARN  [Thread-1937 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-17T20:37:33,609 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe3528f165a5a5eaf with lease ID 0x16d24a360096d367: Processing first storage report for DS-4a1441d3-de19-46ee-8956-edaedf27de80 from datanode DatanodeRegistration(127.0.0.1:42527, datanodeUuid=a93d7341-9dda-4a25-be60-6e7cbeb82954, infoPort=37791, infoSecurePort=0, ipcPort=32891, storageInfo=lv=-57;cid=testClusterID;nsid=707883421;c=1731875851585)
2024-11-17T20:37:33,609 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe3528f165a5a5eaf with lease ID 0x16d24a360096d367: from storage DS-4a1441d3-de19-46ee-8956-edaedf27de80 node DatanodeRegistration(127.0.0.1:42527, datanodeUuid=a93d7341-9dda-4a25-be60-6e7cbeb82954, infoPort=37791, infoSecurePort=0, ipcPort=32891, storageInfo=lv=-57;cid=testClusterID;nsid=707883421;c=1731875851585), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-17T20:37:33,609 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe3528f165a5a5eaf with lease ID 0x16d24a360096d367: Processing first storage report for DS-fdb74203-f338-4f03-a196-c71e925122c8 from datanode DatanodeRegistration(127.0.0.1:42527, datanodeUuid=a93d7341-9dda-4a25-be60-6e7cbeb82954, infoPort=37791, infoSecurePort=0, ipcPort=32891, storageInfo=lv=-57;cid=testClusterID;nsid=707883421;c=1731875851585)
2024-11-17T20:37:33,609 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe3528f165a5a5eaf with lease ID 0x16d24a360096d367: from storage DS-fdb74203-f338-4f03-a196-c71e925122c8 node DatanodeRegistration(127.0.0.1:42527, datanodeUuid=a93d7341-9dda-4a25-be60-6e7cbeb82954, infoPort=37791, infoSecurePort=0, ipcPort=32891, storageInfo=lv=-57;cid=testClusterID;nsid=707883421;c=1731875851585), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-17T20:37:33,717 WARN  [Thread-1984 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1a3196ea-489f-0fa1-8ed8-e2e462983867/cluster_14a128d9-7999-521f-922f-c4fb1ce6abae/data/data3/current/BP-909746748-172.17.0.2-1731875851585/current, will proceed with Du for space computation calculation, 
2024-11-17T20:37:33,717 WARN  [Thread-1985 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1a3196ea-489f-0fa1-8ed8-e2e462983867/cluster_14a128d9-7999-521f-922f-c4fb1ce6abae/data/data4/current/BP-909746748-172.17.0.2-1731875851585/current, will proceed with Du for space computation calculation, 
2024-11-17T20:37:33,739 WARN  [Thread-1960 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-17T20:37:33,741 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf0bd0303235190e4 with lease ID 0x16d24a360096d368: Processing first storage report for DS-cfb5ce05-e87e-4931-a2e3-f714efdf4a33 from datanode DatanodeRegistration(127.0.0.1:44593, datanodeUuid=503c1710-e801-46be-9e8c-b0894925d9a7, infoPort=43885, infoSecurePort=0, ipcPort=45417, storageInfo=lv=-57;cid=testClusterID;nsid=707883421;c=1731875851585)
2024-11-17T20:37:33,741 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf0bd0303235190e4 with lease ID 0x16d24a360096d368: from storage DS-cfb5ce05-e87e-4931-a2e3-f714efdf4a33 node DatanodeRegistration(127.0.0.1:44593, datanodeUuid=503c1710-e801-46be-9e8c-b0894925d9a7, infoPort=43885, infoSecurePort=0, ipcPort=45417, storageInfo=lv=-57;cid=testClusterID;nsid=707883421;c=1731875851585), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-17T20:37:33,741 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf0bd0303235190e4 with lease ID 0x16d24a360096d368: Processing first storage report for DS-2e7e4196-a6ac-45e3-a7b1-5eb6a4e3f3ac from datanode DatanodeRegistration(127.0.0.1:44593, datanodeUuid=503c1710-e801-46be-9e8c-b0894925d9a7, infoPort=43885, infoSecurePort=0, ipcPort=45417, storageInfo=lv=-57;cid=testClusterID;nsid=707883421;c=1731875851585)
2024-11-17T20:37:33,741 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf0bd0303235190e4 with lease ID 0x16d24a360096d368: from storage DS-2e7e4196-a6ac-45e3-a7b1-5eb6a4e3f3ac node DatanodeRegistration(127.0.0.1:44593, datanodeUuid=503c1710-e801-46be-9e8c-b0894925d9a7, infoPort=43885, infoSecurePort=0, ipcPort=45417, storageInfo=lv=-57;cid=testClusterID;nsid=707883421;c=1731875851585), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-17T20:37:33,748 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1a3196ea-489f-0fa1-8ed8-e2e462983867
2024-11-17T20:37:33,751 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1a3196ea-489f-0fa1-8ed8-e2e462983867/cluster_14a128d9-7999-521f-922f-c4fb1ce6abae/zookeeper_0, clientPort=55719, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1a3196ea-489f-0fa1-8ed8-e2e462983867/cluster_14a128d9-7999-521f-922f-c4fb1ce6abae/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1a3196ea-489f-0fa1-8ed8-e2e462983867/cluster_14a128d9-7999-521f-922f-c4fb1ce6abae/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-17T20:37:33,752 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=55719
2024-11-17T20:37:33,752 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T20:37:33,754 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T20:37:33,763 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44593 is added to blk_1073741825_1001 (size=7)
2024-11-17T20:37:33,763 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741825_1001 (size=7)
2024-11-17T20:37:33,764 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f with version=8
2024-11-17T20:37:33,764 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/hbase-staging
2024-11-17T20:37:33,766 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/0b0b0ae8de03:0 server-side Connection retries=45
2024-11-17T20:37:33,766 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-17T20:37:33,766 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-17T20:37:33,766 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-17T20:37:33,766 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-17T20:37:33,766 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-17T20:37:33,766 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-17T20:37:33,766 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-17T20:37:33,767 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33755
2024-11-17T20:37:33,768 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:33755 connecting to ZooKeeper ensemble=127.0.0.1:55719
2024-11-17T20:37:33,803 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-17T20:37:33,803 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-17T20:37:33,804 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-11-17T20:37:33,804 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-17T20:37:33,825 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:33,834 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:33,900 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:337550x0, quorum=127.0.0.1:55719, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-17T20:37:33,900 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:33755-0x1014a8451610000 connected
2024-11-17T20:37:33,978 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T20:37:33,980 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T20:37:33,982 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33755-0x1014a8451610000, quorum=127.0.0.1:55719, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-17T20:37:33,983 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f, hbase.cluster.distributed=false
2024-11-17T20:37:33,985 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33755-0x1014a8451610000, quorum=127.0.0.1:55719, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-17T20:37:33,986 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33755
2024-11-17T20:37:33,986 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33755
2024-11-17T20:37:33,986 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33755
2024-11-17T20:37:33,987 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33755
2024-11-17T20:37:33,987 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33755
2024-11-17T20:37:33,997 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:34,004 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/0b0b0ae8de03:0 server-side Connection retries=45
2024-11-17T20:37:34,004 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-17T20:37:34,004 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-17T20:37:34,004 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-17T20:37:34,004 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-17T20:37:34,004 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-17T20:37:34,004 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-17T20:37:34,004 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-17T20:37:34,005 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44889
2024-11-17T20:37:34,006 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44889 connecting to ZooKeeper ensemble=127.0.0.1:55719
2024-11-17T20:37:34,006 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T20:37:34,008 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T20:37:34,020 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:448890x0, quorum=127.0.0.1:55719, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-17T20:37:34,021 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44889-0x1014a8451610001 connected
2024-11-17T20:37:34,021 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44889-0x1014a8451610001, quorum=127.0.0.1:55719, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-17T20:37:34,021 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-17T20:37:34,021 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-17T20:37:34,022 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44889-0x1014a8451610001, quorum=127.0.0.1:55719, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-17T20:37:34,023 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44889-0x1014a8451610001, quorum=127.0.0.1:55719, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-17T20:37:34,023 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44889
2024-11-17T20:37:34,023 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44889
2024-11-17T20:37:34,023 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44889
2024-11-17T20:37:34,024 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44889
2024-11-17T20:37:34,024 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44889
2024-11-17T20:37:34,038 DEBUG [M:0;0b0b0ae8de03:33755 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;0b0b0ae8de03:33755
2024-11-17T20:37:34,038 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/0b0b0ae8de03,33755,1731875853766
2024-11-17T20:37:34,048 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44889-0x1014a8451610001, quorum=127.0.0.1:55719, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-17T20:37:34,048 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33755-0x1014a8451610000, quorum=127.0.0.1:55719, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-17T20:37:34,049 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33755-0x1014a8451610000, quorum=127.0.0.1:55719, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/0b0b0ae8de03,33755,1731875853766
2024-11-17T20:37:34,059 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33755-0x1014a8451610000, quorum=127.0.0.1:55719, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:37:34,059 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44889-0x1014a8451610001, quorum=127.0.0.1:55719, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-17T20:37:34,059 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44889-0x1014a8451610001, quorum=127.0.0.1:55719, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:37:34,060 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33755-0x1014a8451610000, quorum=127.0.0.1:55719, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-17T20:37:34,060 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/0b0b0ae8de03,33755,1731875853766 from backup master directory
2024-11-17T20:37:34,073 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33755-0x1014a8451610000, quorum=127.0.0.1:55719, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/0b0b0ae8de03,33755,1731875853766
2024-11-17T20:37:34,073 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44889-0x1014a8451610001, quorum=127.0.0.1:55719, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-17T20:37:34,073 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33755-0x1014a8451610000, quorum=127.0.0.1:55719, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-17T20:37:34,073 WARN  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-17T20:37:34,073 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=0b0b0ae8de03,33755,1731875853766
2024-11-17T20:37:34,077 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/hbase.id] with ID: d12fa01c-a924-45c2-aa94-3076bc2f9784
2024-11-17T20:37:34,077 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/.tmp/hbase.id
2024-11-17T20:37:34,082 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44593 is added to blk_1073741826_1002 (size=42)
2024-11-17T20:37:34,082 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741826_1002 (size=42)
2024-11-17T20:37:34,083 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/.tmp/hbase.id]:[hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/hbase.id]
2024-11-17T20:37:34,095 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T20:37:34,095 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-17T20:37:34,096 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms.
2024-11-17T20:37:34,104 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44889-0x1014a8451610001, quorum=127.0.0.1:55719, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:37:34,104 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33755-0x1014a8451610000, quorum=127.0.0.1:55719, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:37:34,118 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741827_1003 (size=196)
2024-11-17T20:37:34,118 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44593 is added to blk_1073741827_1003 (size=196)
2024-11-17T20:37:34,119 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-17T20:37:34,119 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-17T20:37:34,120 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-17T20:37:34,125 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44593 is added to blk_1073741828_1004 (size=1189)
2024-11-17T20:37:34,125 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741828_1004 (size=1189)
2024-11-17T20:37:34,126 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/MasterData/data/master/store
2024-11-17T20:37:34,131 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44593 is added to blk_1073741829_1005 (size=34)
2024-11-17T20:37:34,131 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741829_1005 (size=34)
2024-11-17T20:37:34,132 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T20:37:34,132 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-17T20:37:34,132 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T20:37:34,132 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T20:37:34,132 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-17T20:37:34,132 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T20:37:34,132 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T20:37:34,132 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731875854132Disabling compacts and flushes for region at 1731875854132Disabling writes for close at 1731875854132Writing region close event to WAL at 1731875854132Closed at 1731875854132
2024-11-17T20:37:34,133 WARN  [master/0b0b0ae8de03:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/MasterData/data/master/store/.initializing
2024-11-17T20:37:34,133 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/MasterData/WALs/0b0b0ae8de03,33755,1731875853766
2024-11-17T20:37:34,135 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0b0b0ae8de03%2C33755%2C1731875853766, suffix=, logDir=hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/MasterData/WALs/0b0b0ae8de03,33755,1731875853766, archiveDir=hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/MasterData/oldWALs, maxLogs=10
2024-11-17T20:37:34,136 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0b0b0ae8de03%2C33755%2C1731875853766.1731875854136
2024-11-17T20:37:34,140 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/MasterData/WALs/0b0b0ae8de03,33755,1731875853766/0b0b0ae8de03%2C33755%2C1731875853766.1731875854136
2024-11-17T20:37:34,142 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37791:37791),(127.0.0.1/127.0.0.1:43885:43885)]
2024-11-17T20:37:34,143 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-17T20:37:34,143 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T20:37:34,143 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T20:37:34,143 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T20:37:34,145 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-17T20:37:34,146 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-17T20:37:34,146 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:37:34,147 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T20:37:34,147 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-17T20:37:34,148 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-17T20:37:34,148 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:37:34,148 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-17T20:37:34,148 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-17T20:37:34,149 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-17T20:37:34,149 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:37:34,150 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-17T20:37:34,150 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-17T20:37:34,151 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-17T20:37:34,151 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:37:34,151 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-17T20:37:34,151 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T20:37:34,152 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-17T20:37:34,152 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-17T20:37:34,153 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T20:37:34,153 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T20:37:34,154 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-17T20:37:34,155 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T20:37:34,157 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-17T20:37:34,157 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=736670, jitterRate=-0.06327645480632782}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-17T20:37:34,158 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1731875854143Initializing all the Stores at 1731875854144 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731875854144Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731875854144Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731875854144Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731875854144Cleaning up temporary data from old regions at 1731875854153 (+9 ms)Region opened successfully at 1731875854158 (+5 ms)
2024-11-17T20:37:34,158 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-17T20:37:34,161 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b9e0b37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0b0b0ae8de03/172.17.0.2:0
2024-11-17T20:37:34,162 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-17T20:37:34,162 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-17T20:37:34,162 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-17T20:37:34,162 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-17T20:37:34,163 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-17T20:37:34,163 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-17T20:37:34,163 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-17T20:37:34,166 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-17T20:37:34,167 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33755-0x1014a8451610000, quorum=127.0.0.1:55719, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-17T20:37:34,178 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-17T20:37:34,179 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-17T20:37:34,179 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33755-0x1014a8451610000, quorum=127.0.0.1:55719, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-17T20:37:34,189 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-17T20:37:34,189 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-17T20:37:34,190 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33755-0x1014a8451610000, quorum=127.0.0.1:55719, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-17T20:37:34,199 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-17T20:37:34,200 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33755-0x1014a8451610000, quorum=127.0.0.1:55719, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-17T20:37:34,210 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-17T20:37:34,212 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33755-0x1014a8451610000, quorum=127.0.0.1:55719, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-17T20:37:34,220 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-17T20:37:34,231 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33755-0x1014a8451610000, quorum=127.0.0.1:55719, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-17T20:37:34,231 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44889-0x1014a8451610001, quorum=127.0.0.1:55719, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-17T20:37:34,231 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44889-0x1014a8451610001, quorum=127.0.0.1:55719, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:37:34,231 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33755-0x1014a8451610000, quorum=127.0.0.1:55719, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:37:34,231 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=0b0b0ae8de03,33755,1731875853766, sessionid=0x1014a8451610000, setting cluster-up flag (Was=false)
2024-11-17T20:37:34,252 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44889-0x1014a8451610001, quorum=127.0.0.1:55719, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:37:34,252 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33755-0x1014a8451610000, quorum=127.0.0.1:55719, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:37:34,283 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-17T20:37:34,284 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0b0b0ae8de03,33755,1731875853766
2024-11-17T20:37:34,304 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33755-0x1014a8451610000, quorum=127.0.0.1:55719, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:37:34,304 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44889-0x1014a8451610001, quorum=127.0.0.1:55719, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:37:34,336 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-17T20:37:34,339 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0b0b0ae8de03,33755,1731875853766
2024-11-17T20:37:34,342 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-17T20:37:34,346 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-17T20:37:34,346 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-17T20:37:34,347 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-17T20:37:34,347 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 0b0b0ae8de03,33755,1731875853766
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-17T20:37:34,349 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/0b0b0ae8de03:0, corePoolSize=5, maxPoolSize=5
2024-11-17T20:37:34,349 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/0b0b0ae8de03:0, corePoolSize=5, maxPoolSize=5
2024-11-17T20:37:34,349 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/0b0b0ae8de03:0, corePoolSize=5, maxPoolSize=5
2024-11-17T20:37:34,349 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/0b0b0ae8de03:0, corePoolSize=5, maxPoolSize=5
2024-11-17T20:37:34,349 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/0b0b0ae8de03:0, corePoolSize=10, maxPoolSize=10
2024-11-17T20:37:34,349 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:37:34,349 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/0b0b0ae8de03:0, corePoolSize=2, maxPoolSize=2
2024-11-17T20:37:34,349 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:37:34,350 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731875884350
2024-11-17T20:37:34,350 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-17T20:37:34,350 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-17T20:37:34,350 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-17T20:37:34,350 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-17T20:37:34,350 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-17T20:37:34,350 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-17T20:37:34,351 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-17T20:37:34,351 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-17T20:37:34,351 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-17T20:37:34,351 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-17T20:37:34,351 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-17T20:37:34,351 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-17T20:37:34,351 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-17T20:37:34,352 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-17T20:37:34,352 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/0b0b0ae8de03:0:becomeActiveMaster-HFileCleaner.large.0-1731875854352,5,FailOnTimeoutGroup]
2024-11-17T20:37:34,352 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/0b0b0ae8de03:0:becomeActiveMaster-HFileCleaner.small.0-1731875854352,5,FailOnTimeoutGroup]
2024-11-17T20:37:34,352 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-17T20:37:34,352 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-17T20:37:34,352 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-17T20:37:34,352 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-17T20:37:34,352 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:37:34,353 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-17T20:37:34,359 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44593 is added to blk_1073741831_1007 (size=1321)
2024-11-17T20:37:34,359 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741831_1007 (size=1321)
2024-11-17T20:37:34,360 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-17T20:37:34,360 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f
2024-11-17T20:37:34,371 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44593 is added to blk_1073741832_1008 (size=32)
2024-11-17T20:37:34,371 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741832_1008 (size=32)
2024-11-17T20:37:34,371 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T20:37:34,372 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-17T20:37:34,373 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-17T20:37:34,374 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:37:34,374 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T20:37:34,374 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-17T20:37:34,375 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-17T20:37:34,375 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:37:34,375 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T20:37:34,375 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-17T20:37:34,376 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-17T20:37:34,376 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:37:34,376 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T20:37:34,376 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-17T20:37:34,377 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-17T20:37:34,377 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:37:34,377 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T20:37:34,377 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-17T20:37:34,378 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/hbase/meta/1588230740
2024-11-17T20:37:34,378 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/hbase/meta/1588230740
2024-11-17T20:37:34,379 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-17T20:37:34,379 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-17T20:37:34,379 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-17T20:37:34,380 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-17T20:37:34,382 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-17T20:37:34,383 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=876323, jitterRate=0.11430308222770691}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-17T20:37:34,383 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1731875854371Initializing all the Stores at 1731875854372 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731875854372Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731875854372Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731875854372Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731875854372Cleaning up temporary data from old regions at 1731875854379 (+7 ms)Region opened successfully at 1731875854383 (+4 ms)
2024-11-17T20:37:34,383 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-17T20:37:34,383 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-17T20:37:34,383 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-17T20:37:34,383 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-17T20:37:34,383 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-17T20:37:34,384 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-17T20:37:34,384 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731875854383Disabling compacts and flushes for region at 1731875854383Disabling writes for close at 1731875854383Writing region close event to WAL at 1731875854384 (+1 ms)Closed at 1731875854384
2024-11-17T20:37:34,385 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-17T20:37:34,385 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-17T20:37:34,386 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-17T20:37:34,387 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-17T20:37:34,388 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-17T20:37:34,427 INFO  [RS:0;0b0b0ae8de03:44889 {}] regionserver.HRegionServer(746): ClusterId : d12fa01c-a924-45c2-aa94-3076bc2f9784
2024-11-17T20:37:34,427 DEBUG [RS:0;0b0b0ae8de03:44889 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-17T20:37:34,440 DEBUG [RS:0;0b0b0ae8de03:44889 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-17T20:37:34,440 DEBUG [RS:0;0b0b0ae8de03:44889 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-17T20:37:34,454 DEBUG [RS:0;0b0b0ae8de03:44889 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-17T20:37:34,455 DEBUG [RS:0;0b0b0ae8de03:44889 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39dd1279, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0b0b0ae8de03/172.17.0.2:0
2024-11-17T20:37:34,475 DEBUG [RS:0;0b0b0ae8de03:44889 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;0b0b0ae8de03:44889
2024-11-17T20:37:34,475 INFO  [RS:0;0b0b0ae8de03:44889 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-17T20:37:34,475 INFO  [RS:0;0b0b0ae8de03:44889 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-17T20:37:34,475 DEBUG [RS:0;0b0b0ae8de03:44889 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-17T20:37:34,476 INFO  [RS:0;0b0b0ae8de03:44889 {}] regionserver.HRegionServer(2659): reportForDuty to master=0b0b0ae8de03,33755,1731875853766 with port=44889, startcode=1731875854003
2024-11-17T20:37:34,476 DEBUG [RS:0;0b0b0ae8de03:44889 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-17T20:37:34,478 INFO  [HMaster-EventLoopGroup-14-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38951, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.6 (auth:SIMPLE), service=RegionServerStatusService
2024-11-17T20:37:34,479 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33755 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 0b0b0ae8de03,44889,1731875854003
2024-11-17T20:37:34,479 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33755 {}] master.ServerManager(517): Registering regionserver=0b0b0ae8de03,44889,1731875854003
2024-11-17T20:37:34,480 DEBUG [RS:0;0b0b0ae8de03:44889 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f
2024-11-17T20:37:34,480 DEBUG [RS:0;0b0b0ae8de03:44889 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35449
2024-11-17T20:37:34,480 DEBUG [RS:0;0b0b0ae8de03:44889 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-17T20:37:34,504 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33755-0x1014a8451610000, quorum=127.0.0.1:55719, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-17T20:37:34,505 DEBUG [RS:0;0b0b0ae8de03:44889 {}] zookeeper.ZKUtil(111): regionserver:44889-0x1014a8451610001, quorum=127.0.0.1:55719, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/0b0b0ae8de03,44889,1731875854003
2024-11-17T20:37:34,505 WARN  [RS:0;0b0b0ae8de03:44889 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-17T20:37:34,505 INFO  [RS:0;0b0b0ae8de03:44889 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-17T20:37:34,505 DEBUG [RS:0;0b0b0ae8de03:44889 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/WALs/0b0b0ae8de03,44889,1731875854003
2024-11-17T20:37:34,505 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [0b0b0ae8de03,44889,1731875854003]
2024-11-17T20:37:34,508 INFO  [RS:0;0b0b0ae8de03:44889 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-17T20:37:34,509 INFO  [RS:0;0b0b0ae8de03:44889 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-17T20:37:34,510 INFO  [RS:0;0b0b0ae8de03:44889 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-17T20:37:34,510 INFO  [RS:0;0b0b0ae8de03:44889 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-17T20:37:34,510 INFO  [RS:0;0b0b0ae8de03:44889 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-17T20:37:34,511 INFO  [RS:0;0b0b0ae8de03:44889 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-17T20:37:34,511 INFO  [RS:0;0b0b0ae8de03:44889 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-17T20:37:34,511 DEBUG [RS:0;0b0b0ae8de03:44889 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:37:34,511 DEBUG [RS:0;0b0b0ae8de03:44889 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:37:34,511 DEBUG [RS:0;0b0b0ae8de03:44889 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:37:34,511 DEBUG [RS:0;0b0b0ae8de03:44889 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:37:34,511 DEBUG [RS:0;0b0b0ae8de03:44889 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:37:34,511 DEBUG [RS:0;0b0b0ae8de03:44889 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/0b0b0ae8de03:0, corePoolSize=2, maxPoolSize=2
2024-11-17T20:37:34,511 DEBUG [RS:0;0b0b0ae8de03:44889 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:37:34,511 DEBUG [RS:0;0b0b0ae8de03:44889 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:37:34,511 DEBUG [RS:0;0b0b0ae8de03:44889 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:37:34,511 DEBUG [RS:0;0b0b0ae8de03:44889 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:37:34,511 DEBUG [RS:0;0b0b0ae8de03:44889 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:37:34,511 DEBUG [RS:0;0b0b0ae8de03:44889 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:37:34,511 DEBUG [RS:0;0b0b0ae8de03:44889 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/0b0b0ae8de03:0, corePoolSize=3, maxPoolSize=3
2024-11-17T20:37:34,511 DEBUG [RS:0;0b0b0ae8de03:44889 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0, corePoolSize=3, maxPoolSize=3
2024-11-17T20:37:34,512 INFO  [RS:0;0b0b0ae8de03:44889 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-17T20:37:34,512 INFO  [RS:0;0b0b0ae8de03:44889 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-17T20:37:34,512 INFO  [RS:0;0b0b0ae8de03:44889 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-17T20:37:34,512 INFO  [RS:0;0b0b0ae8de03:44889 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-17T20:37:34,512 INFO  [RS:0;0b0b0ae8de03:44889 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-17T20:37:34,512 INFO  [RS:0;0b0b0ae8de03:44889 {}] hbase.ChoreService(168): Chore ScheduledChore name=0b0b0ae8de03,44889,1731875854003-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-17T20:37:34,527 INFO  [RS:0;0b0b0ae8de03:44889 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-17T20:37:34,527 INFO  [RS:0;0b0b0ae8de03:44889 {}] hbase.ChoreService(168): Chore ScheduledChore name=0b0b0ae8de03,44889,1731875854003-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-17T20:37:34,527 INFO  [RS:0;0b0b0ae8de03:44889 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T20:37:34,527 INFO  [RS:0;0b0b0ae8de03:44889 {}] regionserver.Replication(171): 0b0b0ae8de03,44889,1731875854003 started
2024-11-17T20:37:34,538 WARN  [0b0b0ae8de03:33755 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions.
2024-11-17T20:37:34,540 INFO  [RS:0;0b0b0ae8de03:44889 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T20:37:34,540 INFO  [RS:0;0b0b0ae8de03:44889 {}] regionserver.HRegionServer(1482): Serving as 0b0b0ae8de03,44889,1731875854003, RpcServer on 0b0b0ae8de03/172.17.0.2:44889, sessionid=0x1014a8451610001
2024-11-17T20:37:34,540 DEBUG [RS:0;0b0b0ae8de03:44889 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-17T20:37:34,540 DEBUG [RS:0;0b0b0ae8de03:44889 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 0b0b0ae8de03,44889,1731875854003
2024-11-17T20:37:34,540 DEBUG [RS:0;0b0b0ae8de03:44889 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0b0b0ae8de03,44889,1731875854003'
2024-11-17T20:37:34,541 DEBUG [RS:0;0b0b0ae8de03:44889 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-17T20:37:34,541 DEBUG [RS:0;0b0b0ae8de03:44889 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-17T20:37:34,542 DEBUG [RS:0;0b0b0ae8de03:44889 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-17T20:37:34,542 DEBUG [RS:0;0b0b0ae8de03:44889 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-17T20:37:34,542 DEBUG [RS:0;0b0b0ae8de03:44889 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 0b0b0ae8de03,44889,1731875854003
2024-11-17T20:37:34,542 DEBUG [RS:0;0b0b0ae8de03:44889 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0b0b0ae8de03,44889,1731875854003'
2024-11-17T20:37:34,542 DEBUG [RS:0;0b0b0ae8de03:44889 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-17T20:37:34,542 DEBUG [RS:0;0b0b0ae8de03:44889 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-17T20:37:34,542 DEBUG [RS:0;0b0b0ae8de03:44889 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-17T20:37:34,542 INFO  [RS:0;0b0b0ae8de03:44889 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-17T20:37:34,542 INFO  [RS:0;0b0b0ae8de03:44889 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-17T20:37:34,645 INFO  [RS:0;0b0b0ae8de03:44889 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0b0b0ae8de03%2C44889%2C1731875854003, suffix=, logDir=hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/WALs/0b0b0ae8de03,44889,1731875854003, archiveDir=hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/oldWALs, maxLogs=32
2024-11-17T20:37:34,646 INFO  [RS:0;0b0b0ae8de03:44889 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0b0b0ae8de03%2C44889%2C1731875854003.1731875854646
2024-11-17T20:37:34,656 INFO  [RS:0;0b0b0ae8de03:44889 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/WALs/0b0b0ae8de03,44889,1731875854003/0b0b0ae8de03%2C44889%2C1731875854003.1731875854646
2024-11-17T20:37:34,657 DEBUG [RS:0;0b0b0ae8de03:44889 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37791:37791),(127.0.0.1/127.0.0.1:43885:43885)]
2024-11-17T20:37:34,789 DEBUG [0b0b0ae8de03:33755 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-17T20:37:34,790 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=0b0b0ae8de03,44889,1731875854003
2024-11-17T20:37:34,793 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0b0b0ae8de03,44889,1731875854003, state=OPENING
2024-11-17T20:37:34,810 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-17T20:37:34,821 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44889-0x1014a8451610001, quorum=127.0.0.1:55719, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:37:34,821 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33755-0x1014a8451610000, quorum=127.0.0.1:55719, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:37:34,822 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-17T20:37:34,822 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-17T20:37:34,823 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-17T20:37:34,823 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=0b0b0ae8de03,44889,1731875854003}]
2024-11-17T20:37:34,826 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:34,834 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:34,980 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-17T20:37:34,981 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49853, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-17T20:37:34,985 INFO  [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-17T20:37:34,985 INFO  [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-17T20:37:34,986 INFO  [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0b0b0ae8de03%2C44889%2C1731875854003.meta, suffix=.meta, logDir=hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/WALs/0b0b0ae8de03,44889,1731875854003, archiveDir=hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/oldWALs, maxLogs=32
2024-11-17T20:37:34,987 INFO  [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 0b0b0ae8de03%2C44889%2C1731875854003.meta.1731875854987.meta
2024-11-17T20:37:34,995 INFO  [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/WALs/0b0b0ae8de03,44889,1731875854003/0b0b0ae8de03%2C44889%2C1731875854003.meta.1731875854987.meta
2024-11-17T20:37:34,996 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37791:37791),(127.0.0.1/127.0.0.1:43885:43885)]
2024-11-17T20:37:34,997 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:35,000 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-17T20:37:35,001 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-17T20:37:35,001 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-17T20:37:35,001 INFO  [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-17T20:37:35,001 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-17T20:37:35,001 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T20:37:35,001 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-17T20:37:35,001 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-17T20:37:35,003 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-17T20:37:35,004 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-17T20:37:35,004 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:37:35,004 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T20:37:35,004 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-17T20:37:35,005 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-17T20:37:35,005 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:37:35,006 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T20:37:35,006 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-17T20:37:35,006 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-17T20:37:35,006 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:37:35,007 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T20:37:35,007 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-17T20:37:35,007 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-17T20:37:35,007 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:37:35,008 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T20:37:35,008 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-17T20:37:35,008 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/hbase/meta/1588230740
2024-11-17T20:37:35,009 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/hbase/meta/1588230740
2024-11-17T20:37:35,011 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-17T20:37:35,011 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-17T20:37:35,011 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-17T20:37:35,012 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-17T20:37:35,013 INFO  [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=835353, jitterRate=0.062206923961639404}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-17T20:37:35,013 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-17T20:37:35,014 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1731875855001Writing region info on filesystem at 1731875855001Initializing all the Stores at 1731875855002 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731875855002Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731875855003 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731875855003Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731875855003Cleaning up temporary data from old regions at 1731875855011 (+8 ms)Running coprocessor post-open hooks at 1731875855013 (+2 ms)Region opened successfully at 1731875855014 (+1 ms)
2024-11-17T20:37:35,015 INFO  [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731875854979
2024-11-17T20:37:35,017 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-17T20:37:35,017 INFO  [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-17T20:37:35,018 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=0b0b0ae8de03,44889,1731875854003
2024-11-17T20:37:35,018 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0b0b0ae8de03,44889,1731875854003, state=OPEN
2024-11-17T20:37:35,071 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33755-0x1014a8451610000, quorum=127.0.0.1:55719, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-17T20:37:35,071 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44889-0x1014a8451610001, quorum=127.0.0.1:55719, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-17T20:37:35,071 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=0b0b0ae8de03,44889,1731875854003
2024-11-17T20:37:35,071 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-17T20:37:35,071 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-17T20:37:35,074 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-17T20:37:35,074 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=0b0b0ae8de03,44889,1731875854003 in 248 msec
2024-11-17T20:37:35,077 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-17T20:37:35,077 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 688 msec
2024-11-17T20:37:35,078 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-17T20:37:35,078 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-17T20:37:35,079 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-17T20:37:35,079 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,44889,1731875854003, seqNum=-1]
2024-11-17T20:37:35,080 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-17T20:37:35,081 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-15-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45645, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-17T20:37:35,087 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 742 msec
2024-11-17T20:37:35,087 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731875855087, completionTime=-1
2024-11-17T20:37:35,087 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-17T20:37:35,087 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster...
2024-11-17T20:37:35,090 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1
2024-11-17T20:37:35,090 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731875915090
2024-11-17T20:37:35,090 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731875975090
2024-11-17T20:37:35,090 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 2 msec
2024-11-17T20:37:35,090 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0b0b0ae8de03,33755,1731875853766-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-17T20:37:35,090 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0b0b0ae8de03,33755,1731875853766-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T20:37:35,090 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0b0b0ae8de03,33755,1731875853766-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T20:37:35,090 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-0b0b0ae8de03:33755, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T20:37:35,091 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-17T20:37:35,091 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-17T20:37:35,093 DEBUG [master/0b0b0ae8de03:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-17T20:37:35,095 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.022sec
2024-11-17T20:37:35,095 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-17T20:37:35,095 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-17T20:37:35,096 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-17T20:37:35,096 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-17T20:37:35,096 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-17T20:37:35,096 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0b0b0ae8de03,33755,1731875853766-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-17T20:37:35,096 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0b0b0ae8de03,33755,1731875853766-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-17T20:37:35,098 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-17T20:37:35,098 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-17T20:37:35,099 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0b0b0ae8de03,33755,1731875853766-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T20:37:35,127 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c612ec5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-17T20:37:35,127 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b0b0ae8de03,33755,-1 for getting cluster id
2024-11-17T20:37:35,127 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-17T20:37:35,128 DEBUG [HMaster-EventLoopGroup-14-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd12fa01c-a924-45c2-aa94-3076bc2f9784'
2024-11-17T20:37:35,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-17T20:37:35,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d12fa01c-a924-45c2-aa94-3076bc2f9784"

2024-11-17T20:37:35,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6eff682e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-17T20:37:35,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b0b0ae8de03,33755,-1]
2024-11-17T20:37:35,130 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-17T20:37:35,130 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T20:37:35,131 INFO  [HMaster-EventLoopGroup-14-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44354, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-17T20:37:35,132 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5060cad5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-17T20:37:35,132 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-17T20:37:35,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,44889,1731875854003, seqNum=-1]
2024-11-17T20:37:35,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-17T20:37:35,134 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56198, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-17T20:37:35,136 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=0b0b0ae8de03,33755,1731875853766
2024-11-17T20:37:35,136 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T20:37:35,139 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-17T20:37:35,139 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-17T20:37:35,140 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 0b0b0ae8de03,33755,1731875853766
2024-11-17T20:37:35,140 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2714f3ca
2024-11-17T20:37:35,140 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-17T20:37:35,142 INFO  [HMaster-EventLoopGroup-14-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44364, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-17T20:37:35,142 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33755 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-11-17T20:37:35,142 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33755 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-11-17T20:37:35,142 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33755 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-17T20:37:35,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33755 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRolling
2024-11-17T20:37:35,146 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-11-17T20:37:35,146 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:37:35,146 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33755 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRolling"
 procId is: 4
2024-11-17T20:37:35,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33755 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-17T20:37:35,147 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-17T20:37:35,157 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44593 is added to blk_1073741835_1011 (size=381)
2024-11-17T20:37:35,158 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741835_1011 (size=381)
2024-11-17T20:37:35,160 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 80ad337678f5ab9b11d51704f5980d4a, NAME => 'TestLogRolling-testLogRolling,,1731875855142.80ad337678f5ab9b11d51704f5980d4a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f
2024-11-17T20:37:35,171 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44593 is added to blk_1073741836_1012 (size=64)
2024-11-17T20:37:35,171 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741836_1012 (size=64)
2024-11-17T20:37:35,172 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1731875855142.80ad337678f5ab9b11d51704f5980d4a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T20:37:35,172 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1722): Closing 80ad337678f5ab9b11d51704f5980d4a, disabling compactions & flushes
2024-11-17T20:37:35,172 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1731875855142.80ad337678f5ab9b11d51704f5980d4a.
2024-11-17T20:37:35,172 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1731875855142.80ad337678f5ab9b11d51704f5980d4a.
2024-11-17T20:37:35,172 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1731875855142.80ad337678f5ab9b11d51704f5980d4a. after waiting 0 ms
2024-11-17T20:37:35,172 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1731875855142.80ad337678f5ab9b11d51704f5980d4a.
2024-11-17T20:37:35,172 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1731875855142.80ad337678f5ab9b11d51704f5980d4a.
2024-11-17T20:37:35,172 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for 80ad337678f5ab9b11d51704f5980d4a:
Waiting for close lock at 1731875855172Disabling compacts and flushes for region at 1731875855172Disabling writes for close at 1731875855172Writing region close event to WAL at 1731875855172Closed at 1731875855172
2024-11-17T20:37:35,174 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ADD_TO_META
2024-11-17T20:37:35,174 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRolling,,1731875855142.80ad337678f5ab9b11d51704f5980d4a.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1731875855174"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731875855174"}]},"ts":"1731875855174"}
2024-11-17T20:37:35,176 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-17T20:37:35,178 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-17T20:37:35,178 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731875855178"}]},"ts":"1731875855178"}
2024-11-17T20:37:35,180 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRolling, state=ENABLING in hbase:meta
2024-11-17T20:37:35,180 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=80ad337678f5ab9b11d51704f5980d4a, ASSIGN}]
2024-11-17T20:37:35,182 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=80ad337678f5ab9b11d51704f5980d4a, ASSIGN
2024-11-17T20:37:35,183 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=80ad337678f5ab9b11d51704f5980d4a, ASSIGN; state=OFFLINE, location=0b0b0ae8de03,44889,1731875854003; forceNewPlan=false, retain=false
2024-11-17T20:37:35,334 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=80ad337678f5ab9b11d51704f5980d4a, regionState=OPENING, regionLocation=0b0b0ae8de03,44889,1731875854003
2024-11-17T20:37:35,336 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=80ad337678f5ab9b11d51704f5980d4a, ASSIGN because future has completed
2024-11-17T20:37:35,337 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 80ad337678f5ab9b11d51704f5980d4a, server=0b0b0ae8de03,44889,1731875854003}]
2024-11-17T20:37:35,493 INFO  [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,,1731875855142.80ad337678f5ab9b11d51704f5980d4a.
2024-11-17T20:37:35,494 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 80ad337678f5ab9b11d51704f5980d4a, NAME => 'TestLogRolling-testLogRolling,,1731875855142.80ad337678f5ab9b11d51704f5980d4a.', STARTKEY => '', ENDKEY => ''}
2024-11-17T20:37:35,494 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling 80ad337678f5ab9b11d51704f5980d4a
2024-11-17T20:37:35,494 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1731875855142.80ad337678f5ab9b11d51704f5980d4a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T20:37:35,494 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 80ad337678f5ab9b11d51704f5980d4a
2024-11-17T20:37:35,494 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 80ad337678f5ab9b11d51704f5980d4a
2024-11-17T20:37:35,495 INFO  [StoreOpener-80ad337678f5ab9b11d51704f5980d4a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 80ad337678f5ab9b11d51704f5980d4a 
2024-11-17T20:37:35,497 INFO  [StoreOpener-80ad337678f5ab9b11d51704f5980d4a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 80ad337678f5ab9b11d51704f5980d4a columnFamilyName info
2024-11-17T20:37:35,497 DEBUG [StoreOpener-80ad337678f5ab9b11d51704f5980d4a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:37:35,497 INFO  [StoreOpener-80ad337678f5ab9b11d51704f5980d4a-1 {}] regionserver.HStore(327): Store=80ad337678f5ab9b11d51704f5980d4a/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-17T20:37:35,498 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 80ad337678f5ab9b11d51704f5980d4a
2024-11-17T20:37:35,498 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a
2024-11-17T20:37:35,499 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a
2024-11-17T20:37:35,499 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 80ad337678f5ab9b11d51704f5980d4a
2024-11-17T20:37:35,499 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 80ad337678f5ab9b11d51704f5980d4a
2024-11-17T20:37:35,501 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 80ad337678f5ab9b11d51704f5980d4a
2024-11-17T20:37:35,505 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-17T20:37:35,506 INFO  [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 80ad337678f5ab9b11d51704f5980d4a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=802379, jitterRate=0.020278021693229675}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-17T20:37:35,506 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 80ad337678f5ab9b11d51704f5980d4a
2024-11-17T20:37:35,506 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 80ad337678f5ab9b11d51704f5980d4a:
Running coprocessor pre-open hook at 1731875855494Writing region info on filesystem at 1731875855494Initializing all the Stores at 1731875855495 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731875855495Cleaning up temporary data from old regions at 1731875855499 (+4 ms)Running coprocessor post-open hooks at 1731875855506 (+7 ms)Region opened successfully at 1731875855506
2024-11-17T20:37:35,507 INFO  [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,,1731875855142.80ad337678f5ab9b11d51704f5980d4a., pid=6, masterSystemTime=1731875855490
2024-11-17T20:37:35,510 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,,1731875855142.80ad337678f5ab9b11d51704f5980d4a.
2024-11-17T20:37:35,510 INFO  [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,,1731875855142.80ad337678f5ab9b11d51704f5980d4a.
2024-11-17T20:37:35,511 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=80ad337678f5ab9b11d51704f5980d4a, regionState=OPEN, openSeqNum=2, regionLocation=0b0b0ae8de03,44889,1731875854003
2024-11-17T20:37:35,513 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 80ad337678f5ab9b11d51704f5980d4a, server=0b0b0ae8de03,44889,1731875854003 because future has completed
2024-11-17T20:37:35,517 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-17T20:37:35,517 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 80ad337678f5ab9b11d51704f5980d4a, server=0b0b0ae8de03,44889,1731875854003 in 177 msec
2024-11-17T20:37:35,521 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-17T20:37:35,521 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=80ad337678f5ab9b11d51704f5980d4a, ASSIGN in 337 msec
2024-11-17T20:37:35,522 INFO  [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-17T20:37:35,522 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731875855522"}]},"ts":"1731875855522"}
2024-11-17T20:37:35,525 INFO  [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRolling, state=ENABLED in hbase:meta
2024-11-17T20:37:35,526 INFO  [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_POST_OPERATION
2024-11-17T20:37:35,529 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRolling in 384 msec
2024-11-17T20:37:35,827 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:35,835 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:35,994 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:37:35,994 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:37:35,994 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:37:35,995 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:37:35,995 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:37:35,998 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:36,000 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:37:36,027 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:37:36,027 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:37:36,027 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:37:36,028 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:37:36,028 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:37:36,028 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:37:36,033 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:37:36,034 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:37:36,034 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:37:36,037 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:37:36,543 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-17T20:37:36,544 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:37:36,544 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:37:36,544 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:37:36,544 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:37:36,545 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:37:36,545 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:37:36,566 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:37:36,566 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:37:36,567 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:37:36,567 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:37:36,567 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:37:36,568 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:37:36,574 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:37:36,574 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:37:36,574 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:37:36,578 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:37:36,828 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:36,835 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:36,998 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:37,828 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:37,836 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:37,999 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:38,829 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:38,837 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:39,000 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:39,830 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:39,837 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:40,000 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:40,508 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-17T20:37:40,509 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRolling'
2024-11-17T20:37:40,830 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:40,838 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:41,001 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:41,831 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:41,838 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:42,001 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:42,831 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:42,838 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:43,001 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:43,803 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-17T20:37:43,803 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-11-17T20:37:43,804 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-17T20:37:43,804 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-11-17T20:37:43,804 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-11-17T20:37:43,804 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer
2024-11-17T20:37:43,804 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling
2024-11-17T20:37:43,804 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling Metrics about Tables on a single HBase RegionServer
2024-11-17T20:37:43,832 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:43,839 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:44,002 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:44,832 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:44,839 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:45,002 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:45,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33755 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-17T20:37:45,198 INFO  [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRolling completed
2024-11-17T20:37:45,198 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRolling,, stopping at row=TestLogRolling-testLogRolling ,, for max=2147483647 with caching=100
2024-11-17T20:37:45,202 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRolling
2024-11-17T20:37:45,202 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRolling,,1731875855142.80ad337678f5ab9b11d51704f5980d4a.
2024-11-17T20:37:45,207 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testLogRolling,,1731875855142.80ad337678f5ab9b11d51704f5980d4a., hostname=0b0b0ae8de03,44889,1731875854003, seqNum=2]
2024-11-17T20:37:45,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] regionserver.HRegion(8855): Flush requested on 80ad337678f5ab9b11d51704f5980d4a
2024-11-17T20:37:45,221 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 80ad337678f5ab9b11d51704f5980d4a 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-17T20:37:45,239 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/.tmp/info/41d6987b2c5a4a219fadc9cb5d625183 is 1080, key is row0001/info:/1731875865209/Put/seqid=0
2024-11-17T20:37:45,244 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44593 is added to blk_1073741837_1013 (size=12509)
2024-11-17T20:37:45,244 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741837_1013 (size=12509)
2024-11-17T20:37:45,245 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/.tmp/info/41d6987b2c5a4a219fadc9cb5d625183
2024-11-17T20:37:45,252 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/.tmp/info/41d6987b2c5a4a219fadc9cb5d625183 as hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/41d6987b2c5a4a219fadc9cb5d625183
2024-11-17T20:37:45,258 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/41d6987b2c5a4a219fadc9cb5d625183, entries=7, sequenceid=11, filesize=12.2 K
2024-11-17T20:37:45,259 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=17.86 KB/18292 for 80ad337678f5ab9b11d51704f5980d4a in 38ms, sequenceid=11, compaction requested=false
2024-11-17T20:37:45,259 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 80ad337678f5ab9b11d51704f5980d4a:

2024-11-17T20:37:45,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] regionserver.HRegion(8855): Flush requested on 80ad337678f5ab9b11d51704f5980d4a
2024-11-17T20:37:45,260 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 80ad337678f5ab9b11d51704f5980d4a 1/1 column families, dataSize=18.91 KB heapSize=20.50 KB
2024-11-17T20:37:45,265 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/.tmp/info/0f005012635d42e3908daa1d9ac8589e is 1080, key is row0008/info:/1731875865222/Put/seqid=0
2024-11-17T20:37:45,270 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44593 is added to blk_1073741838_1014 (size=24376)
2024-11-17T20:37:45,270 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741838_1014 (size=24376)
2024-11-17T20:37:45,271 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=18.91 KB at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/.tmp/info/0f005012635d42e3908daa1d9ac8589e
2024-11-17T20:37:45,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/.tmp/info/0f005012635d42e3908daa1d9ac8589e as hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/0f005012635d42e3908daa1d9ac8589e
2024-11-17T20:37:45,282 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/0f005012635d42e3908daa1d9ac8589e, entries=18, sequenceid=32, filesize=23.8 K
2024-11-17T20:37:45,283 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~18.91 KB/19368, heapSize ~20.48 KB/20976, currentSize=7.36 KB/7532 for 80ad337678f5ab9b11d51704f5980d4a in 23ms, sequenceid=32, compaction requested=false
2024-11-17T20:37:45,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 80ad337678f5ab9b11d51704f5980d4a:

2024-11-17T20:37:45,283 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=36.0 K, sizeToCheck=16.0 K
2024-11-17T20:37:45,283 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-17T20:37:45,283 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/0f005012635d42e3908daa1d9ac8589e because midkey is the same as first or last row
2024-11-17T20:37:45,833 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:45,840 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:46,003 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:46,834 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:46,841 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:47,004 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:47,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] regionserver.HRegion(8855): Flush requested on 80ad337678f5ab9b11d51704f5980d4a
2024-11-17T20:37:47,279 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 80ad337678f5ab9b11d51704f5980d4a 1/1 column families, dataSize=8.41 KB heapSize=9.25 KB
2024-11-17T20:37:47,287 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/.tmp/info/5ae1da2e369f493292bd376e3a11154e is 1080, key is row0026/info:/1731875865261/Put/seqid=0
2024-11-17T20:37:47,292 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741839_1015 (size=13586)
2024-11-17T20:37:47,292 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44593 is added to blk_1073741839_1015 (size=13586)
2024-11-17T20:37:47,293 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.41 KB at sequenceid=43 (bloomFilter=true), to=hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/.tmp/info/5ae1da2e369f493292bd376e3a11154e
2024-11-17T20:37:47,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/.tmp/info/5ae1da2e369f493292bd376e3a11154e as hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/5ae1da2e369f493292bd376e3a11154e
2024-11-17T20:37:47,307 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/5ae1da2e369f493292bd376e3a11154e, entries=8, sequenceid=43, filesize=13.3 K
2024-11-17T20:37:47,307 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~8.41 KB/8608, heapSize ~9.23 KB/9456, currentSize=11.56 KB/11836 for 80ad337678f5ab9b11d51704f5980d4a in 28ms, sequenceid=43, compaction requested=true
2024-11-17T20:37:47,307 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 80ad337678f5ab9b11d51704f5980d4a:

2024-11-17T20:37:47,307 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=49.3 K, sizeToCheck=16.0 K
2024-11-17T20:37:47,307 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-17T20:37:47,307 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/0f005012635d42e3908daa1d9ac8589e because midkey is the same as first or last row
2024-11-17T20:37:47,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 80ad337678f5ab9b11d51704f5980d4a:info, priority=-2147483648, current under compaction store size is 1
2024-11-17T20:37:47,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-17T20:37:47,308 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-17T20:37:47,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] regionserver.HRegion(8855): Flush requested on 80ad337678f5ab9b11d51704f5980d4a
2024-11-17T20:37:47,309 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 80ad337678f5ab9b11d51704f5980d4a 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB
2024-11-17T20:37:47,309 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 50471 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-17T20:37:47,309 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.HStore(1541): 80ad337678f5ab9b11d51704f5980d4a/info is initiating minor compaction (all files)
2024-11-17T20:37:47,309 INFO  [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 80ad337678f5ab9b11d51704f5980d4a/info in TestLogRolling-testLogRolling,,1731875855142.80ad337678f5ab9b11d51704f5980d4a.
2024-11-17T20:37:47,309 INFO  [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/41d6987b2c5a4a219fadc9cb5d625183, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/0f005012635d42e3908daa1d9ac8589e, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/5ae1da2e369f493292bd376e3a11154e] into tmpdir=hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/.tmp, totalSize=49.3 K
2024-11-17T20:37:47,310 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41d6987b2c5a4a219fadc9cb5d625183, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731875865209
2024-11-17T20:37:47,310 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f005012635d42e3908daa1d9ac8589e, keycount=18, bloomtype=ROW, size=23.8 K, encoding=NONE, compression=NONE, seqNum=32, earliestPutTs=1731875865222
2024-11-17T20:37:47,311 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ae1da2e369f493292bd376e3a11154e, keycount=8, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1731875865261
2024-11-17T20:37:47,312 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/.tmp/info/9a397c9afce54727a86d220269ddc63e is 1080, key is row0034/info:/1731875867282/Put/seqid=0
2024-11-17T20:37:47,317 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741840_1016 (size=17894)
2024-11-17T20:37:47,317 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44593 is added to blk_1073741840_1016 (size=17894)
2024-11-17T20:37:47,318 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/.tmp/info/9a397c9afce54727a86d220269ddc63e
2024-11-17T20:37:47,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/.tmp/info/9a397c9afce54727a86d220269ddc63e as hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/9a397c9afce54727a86d220269ddc63e
2024-11-17T20:37:47,331 INFO  [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 80ad337678f5ab9b11d51704f5980d4a#info#compaction#59 average throughput is 16.93 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-17T20:37:47,331 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/.tmp/info/6f4eb6543f0d462b8d5ba06354f5e6a9 is 1080, key is row0001/info:/1731875865209/Put/seqid=0
2024-11-17T20:37:47,332 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/9a397c9afce54727a86d220269ddc63e, entries=12, sequenceid=58, filesize=17.5 K
2024-11-17T20:37:47,333 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=10.51 KB/10760 for 80ad337678f5ab9b11d51704f5980d4a in 24ms, sequenceid=58, compaction requested=false
2024-11-17T20:37:47,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 80ad337678f5ab9b11d51704f5980d4a:

2024-11-17T20:37:47,333 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=66.8 K, sizeToCheck=16.0 K
2024-11-17T20:37:47,333 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-17T20:37:47,333 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/0f005012635d42e3908daa1d9ac8589e because midkey is the same as first or last row
2024-11-17T20:37:47,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] regionserver.HRegion(8855): Flush requested on 80ad337678f5ab9b11d51704f5980d4a
2024-11-17T20:37:47,334 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 80ad337678f5ab9b11d51704f5980d4a 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB
2024-11-17T20:37:47,336 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44593 is added to blk_1073741841_1017 (size=40670)
2024-11-17T20:37:47,336 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741841_1017 (size=40670)
2024-11-17T20:37:47,339 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/.tmp/info/48a8ae3e483c4fdb8da246b88d967783 is 1080, key is row0046/info:/1731875867310/Put/seqid=0
2024-11-17T20:37:47,343 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/.tmp/info/6f4eb6543f0d462b8d5ba06354f5e6a9 as hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/6f4eb6543f0d462b8d5ba06354f5e6a9
2024-11-17T20:37:47,349 INFO  [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 80ad337678f5ab9b11d51704f5980d4a/info of 80ad337678f5ab9b11d51704f5980d4a into 6f4eb6543f0d462b8d5ba06354f5e6a9(size=39.7 K), total size for store is 57.2 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-17T20:37:47,349 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 80ad337678f5ab9b11d51704f5980d4a:

2024-11-17T20:37:47,350 INFO  [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1731875855142.80ad337678f5ab9b11d51704f5980d4a., storeName=80ad337678f5ab9b11d51704f5980d4a/info, priority=13, startTime=1731875867308; duration=0sec
2024-11-17T20:37:47,350 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=57.2 K, sizeToCheck=16.0 K
2024-11-17T20:37:47,350 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-17T20:37:47,350 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/6f4eb6543f0d462b8d5ba06354f5e6a9 because midkey is the same as first or last row
2024-11-17T20:37:47,350 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=57.2 K, sizeToCheck=16.0 K
2024-11-17T20:37:47,350 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-17T20:37:47,350 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/6f4eb6543f0d462b8d5ba06354f5e6a9 because midkey is the same as first or last row
2024-11-17T20:37:47,350 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=57.2 K, sizeToCheck=16.0 K
2024-11-17T20:37:47,350 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-17T20:37:47,350 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/6f4eb6543f0d462b8d5ba06354f5e6a9 because midkey is the same as first or last row
2024-11-17T20:37:47,350 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-17T20:37:47,350 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 80ad337678f5ab9b11d51704f5980d4a:info
2024-11-17T20:37:47,353 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741842_1018 (size=16817)
2024-11-17T20:37:47,354 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44593 is added to blk_1073741842_1018 (size=16817)
2024-11-17T20:37:47,354 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=72 (bloomFilter=true), to=hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/.tmp/info/48a8ae3e483c4fdb8da246b88d967783
2024-11-17T20:37:47,359 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/.tmp/info/48a8ae3e483c4fdb8da246b88d967783 as hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/48a8ae3e483c4fdb8da246b88d967783
2024-11-17T20:37:47,364 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/48a8ae3e483c4fdb8da246b88d967783, entries=11, sequenceid=72, filesize=16.4 K
2024-11-17T20:37:47,365 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=8.41 KB/8608 for 80ad337678f5ab9b11d51704f5980d4a in 31ms, sequenceid=72, compaction requested=true
2024-11-17T20:37:47,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 80ad337678f5ab9b11d51704f5980d4a:

2024-11-17T20:37:47,366 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=73.6 K, sizeToCheck=16.0 K
2024-11-17T20:37:47,366 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-17T20:37:47,366 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/6f4eb6543f0d462b8d5ba06354f5e6a9 because midkey is the same as first or last row
2024-11-17T20:37:47,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 80ad337678f5ab9b11d51704f5980d4a:info, priority=-2147483648, current under compaction store size is 1
2024-11-17T20:37:47,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-17T20:37:47,366 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-17T20:37:47,367 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 75381 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-17T20:37:47,367 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.HStore(1541): 80ad337678f5ab9b11d51704f5980d4a/info is initiating minor compaction (all files)
2024-11-17T20:37:47,367 INFO  [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 80ad337678f5ab9b11d51704f5980d4a/info in TestLogRolling-testLogRolling,,1731875855142.80ad337678f5ab9b11d51704f5980d4a.
2024-11-17T20:37:47,367 INFO  [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/6f4eb6543f0d462b8d5ba06354f5e6a9, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/9a397c9afce54727a86d220269ddc63e, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/48a8ae3e483c4fdb8da246b88d967783] into tmpdir=hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/.tmp, totalSize=73.6 K
2024-11-17T20:37:47,368 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f4eb6543f0d462b8d5ba06354f5e6a9, keycount=33, bloomtype=ROW, size=39.7 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1731875865209
2024-11-17T20:37:47,368 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a397c9afce54727a86d220269ddc63e, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1731875867282
2024-11-17T20:37:47,369 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48a8ae3e483c4fdb8da246b88d967783, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=72, earliestPutTs=1731875867310
2024-11-17T20:37:47,379 INFO  [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 80ad337678f5ab9b11d51704f5980d4a#info#compaction#61 average throughput is 28.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-17T20:37:47,379 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/.tmp/info/aa9508865dcf4e518503a2a045bb09c1 is 1080, key is row0001/info:/1731875865209/Put/seqid=0
2024-11-17T20:37:47,384 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741843_1019 (size=65612)
2024-11-17T20:37:47,384 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44593 is added to blk_1073741843_1019 (size=65612)
2024-11-17T20:37:47,390 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/.tmp/info/aa9508865dcf4e518503a2a045bb09c1 as hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/aa9508865dcf4e518503a2a045bb09c1
2024-11-17T20:37:47,396 INFO  [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 80ad337678f5ab9b11d51704f5980d4a/info of 80ad337678f5ab9b11d51704f5980d4a into aa9508865dcf4e518503a2a045bb09c1(size=64.1 K), total size for store is 64.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-17T20:37:47,396 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 80ad337678f5ab9b11d51704f5980d4a:

2024-11-17T20:37:47,396 INFO  [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1731875855142.80ad337678f5ab9b11d51704f5980d4a., storeName=80ad337678f5ab9b11d51704f5980d4a/info, priority=13, startTime=1731875867366; duration=0sec
2024-11-17T20:37:47,396 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=64.1 K, sizeToCheck=16.0 K
2024-11-17T20:37:47,396 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-17T20:37:47,396 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/aa9508865dcf4e518503a2a045bb09c1 because midkey is the same as first or last row
2024-11-17T20:37:47,396 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=64.1 K, sizeToCheck=16.0 K
2024-11-17T20:37:47,396 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-17T20:37:47,396 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/aa9508865dcf4e518503a2a045bb09c1 because midkey is the same as first or last row
2024-11-17T20:37:47,397 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=64.1 K, sizeToCheck=16.0 K
2024-11-17T20:37:47,397 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-17T20:37:47,397 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/aa9508865dcf4e518503a2a045bb09c1 because midkey is the same as first or last row
2024-11-17T20:37:47,397 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-17T20:37:47,397 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 80ad337678f5ab9b11d51704f5980d4a:info
2024-11-17T20:37:47,834 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:47,841 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:48,004 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:48,835 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:48,842 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:49,005 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:49,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] regionserver.HRegion(8855): Flush requested on 80ad337678f5ab9b11d51704f5980d4a
2024-11-17T20:37:49,354 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 80ad337678f5ab9b11d51704f5980d4a 1/1 column families, dataSize=9.46 KB heapSize=10.38 KB
2024-11-17T20:37:49,360 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/.tmp/info/7a32ae70e9014f1aab838b4bfd775e6f is 1080, key is row0057/info:/1731875867336/Put/seqid=0
2024-11-17T20:37:49,366 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44593 is added to blk_1073741844_1020 (size=14663)
2024-11-17T20:37:49,366 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741844_1020 (size=14663)
2024-11-17T20:37:49,367 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=9.46 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/.tmp/info/7a32ae70e9014f1aab838b4bfd775e6f
2024-11-17T20:37:49,373 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/.tmp/info/7a32ae70e9014f1aab838b4bfd775e6f as hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/7a32ae70e9014f1aab838b4bfd775e6f
2024-11-17T20:37:49,380 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/7a32ae70e9014f1aab838b4bfd775e6f, entries=9, sequenceid=86, filesize=14.3 K
2024-11-17T20:37:49,382 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~9.46 KB/9684, heapSize ~10.36 KB/10608, currentSize=10.51 KB/10760 for 80ad337678f5ab9b11d51704f5980d4a in 28ms, sequenceid=86, compaction requested=false
2024-11-17T20:37:49,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 80ad337678f5ab9b11d51704f5980d4a:

2024-11-17T20:37:49,382 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=78.4 K, sizeToCheck=16.0 K
2024-11-17T20:37:49,382 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-17T20:37:49,382 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/aa9508865dcf4e518503a2a045bb09c1 because midkey is the same as first or last row
2024-11-17T20:37:49,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] regionserver.HRegion(8855): Flush requested on 80ad337678f5ab9b11d51704f5980d4a
2024-11-17T20:37:49,384 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 80ad337678f5ab9b11d51704f5980d4a 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB
2024-11-17T20:37:49,389 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/.tmp/info/7ed4d2f96d2f42d48d77257feaf87f64 is 1080, key is row0066/info:/1731875869356/Put/seqid=0
2024-11-17T20:37:49,402 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44593 is added to blk_1073741845_1021 (size=16817)
2024-11-17T20:37:49,403 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741845_1021 (size=16817)
2024-11-17T20:37:49,404 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/.tmp/info/7ed4d2f96d2f42d48d77257feaf87f64
2024-11-17T20:37:49,411 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/.tmp/info/7ed4d2f96d2f42d48d77257feaf87f64 as hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/7ed4d2f96d2f42d48d77257feaf87f64
2024-11-17T20:37:49,417 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/7ed4d2f96d2f42d48d77257feaf87f64, entries=11, sequenceid=100, filesize=16.4 K
2024-11-17T20:37:49,418 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=15.76 KB/16140 for 80ad337678f5ab9b11d51704f5980d4a in 35ms, sequenceid=100, compaction requested=true
2024-11-17T20:37:49,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 80ad337678f5ab9b11d51704f5980d4a:

2024-11-17T20:37:49,418 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=94.8 K, sizeToCheck=16.0 K
2024-11-17T20:37:49,418 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-17T20:37:49,418 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/aa9508865dcf4e518503a2a045bb09c1 because midkey is the same as first or last row
2024-11-17T20:37:49,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 80ad337678f5ab9b11d51704f5980d4a:info, priority=-2147483648, current under compaction store size is 1
2024-11-17T20:37:49,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-17T20:37:49,419 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-17T20:37:49,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] regionserver.HRegion(8855): Flush requested on 80ad337678f5ab9b11d51704f5980d4a
2024-11-17T20:37:49,420 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 97092 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-17T20:37:49,420 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.HStore(1541): 80ad337678f5ab9b11d51704f5980d4a/info is initiating minor compaction (all files)
2024-11-17T20:37:49,420 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 80ad337678f5ab9b11d51704f5980d4a 1/1 column families, dataSize=16.81 KB heapSize=18.25 KB
2024-11-17T20:37:49,420 INFO  [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 80ad337678f5ab9b11d51704f5980d4a/info in TestLogRolling-testLogRolling,,1731875855142.80ad337678f5ab9b11d51704f5980d4a.
2024-11-17T20:37:49,420 INFO  [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/aa9508865dcf4e518503a2a045bb09c1, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/7a32ae70e9014f1aab838b4bfd775e6f, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/7ed4d2f96d2f42d48d77257feaf87f64] into tmpdir=hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/.tmp, totalSize=94.8 K
2024-11-17T20:37:49,421 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa9508865dcf4e518503a2a045bb09c1, keycount=56, bloomtype=ROW, size=64.1 K, encoding=NONE, compression=NONE, seqNum=72, earliestPutTs=1731875865209
2024-11-17T20:37:49,421 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a32ae70e9014f1aab838b4bfd775e6f, keycount=9, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731875867336
2024-11-17T20:37:49,422 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ed4d2f96d2f42d48d77257feaf87f64, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731875869356
2024-11-17T20:37:49,426 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/.tmp/info/15b37024d5e44bb39aa4456aabc889a6 is 1080, key is row0077/info:/1731875869385/Put/seqid=0
2024-11-17T20:37:49,447 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741846_1022 (size=22222)
2024-11-17T20:37:49,447 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44593 is added to blk_1073741846_1022 (size=22222)
2024-11-17T20:37:49,449 INFO  [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 80ad337678f5ab9b11d51704f5980d4a#info#compaction#65 average throughput is 26.00 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-17T20:37:49,449 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.81 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/.tmp/info/15b37024d5e44bb39aa4456aabc889a6
2024-11-17T20:37:49,449 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/.tmp/info/7a56768371864672b2334aa7f10cf37b is 1080, key is row0001/info:/1731875865209/Put/seqid=0
2024-11-17T20:37:49,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/.tmp/info/15b37024d5e44bb39aa4456aabc889a6 as hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/15b37024d5e44bb39aa4456aabc889a6
2024-11-17T20:37:49,458 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44593 is added to blk_1073741847_1023 (size=87327)
2024-11-17T20:37:49,458 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741847_1023 (size=87327)
2024-11-17T20:37:49,461 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/15b37024d5e44bb39aa4456aabc889a6, entries=16, sequenceid=119, filesize=21.7 K
2024-11-17T20:37:49,462 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~16.81 KB/17216, heapSize ~18.23 KB/18672, currentSize=4.20 KB/4304 for 80ad337678f5ab9b11d51704f5980d4a in 42ms, sequenceid=119, compaction requested=false
2024-11-17T20:37:49,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 80ad337678f5ab9b11d51704f5980d4a:

2024-11-17T20:37:49,463 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=116.5 K, sizeToCheck=16.0 K
2024-11-17T20:37:49,463 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-17T20:37:49,463 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/aa9508865dcf4e518503a2a045bb09c1 because midkey is the same as first or last row
2024-11-17T20:37:49,466 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/.tmp/info/7a56768371864672b2334aa7f10cf37b as hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/7a56768371864672b2334aa7f10cf37b
2024-11-17T20:37:49,472 INFO  [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 80ad337678f5ab9b11d51704f5980d4a/info of 80ad337678f5ab9b11d51704f5980d4a into 7a56768371864672b2334aa7f10cf37b(size=85.3 K), total size for store is 107.0 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-17T20:37:49,472 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 80ad337678f5ab9b11d51704f5980d4a:

2024-11-17T20:37:49,472 INFO  [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1731875855142.80ad337678f5ab9b11d51704f5980d4a., storeName=80ad337678f5ab9b11d51704f5980d4a/info, priority=13, startTime=1731875869418; duration=0sec
2024-11-17T20:37:49,472 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=107.0 K, sizeToCheck=16.0 K
2024-11-17T20:37:49,472 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-17T20:37:49,472 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=107.0 K, sizeToCheck=16.0 K
2024-11-17T20:37:49,472 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-17T20:37:49,473 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=107.0 K, sizeToCheck=16.0 K
2024-11-17T20:37:49,473 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-17T20:37:49,473 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.CompactSplit(239): Splitting TestLogRolling-testLogRolling,,1731875855142.80ad337678f5ab9b11d51704f5980d4a., compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-17T20:37:49,474 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-17T20:37:49,474 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 80ad337678f5ab9b11d51704f5980d4a:info
2024-11-17T20:37:49,475 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33755 {}] assignment.AssignmentManager(1355): Split request from 0b0b0ae8de03,44889,1731875854003, parent={ENCODED => 80ad337678f5ab9b11d51704f5980d4a, NAME => 'TestLogRolling-testLogRolling,,1731875855142.80ad337678f5ab9b11d51704f5980d4a.', STARTKEY => '', ENDKEY => ''}, splitKey=row0062
2024-11-17T20:37:49,480 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33755 {}] assignment.SplitTableRegionProcedure(223): Splittable=true state=OPEN, location=0b0b0ae8de03,44889,1731875854003
2024-11-17T20:37:49,485 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33755 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=80ad337678f5ab9b11d51704f5980d4a, daughterA=5868b317f196d2a5e51ca2d7baef0437, daughterB=e7b653549005a2a5c4cce0702ee0803c
2024-11-17T20:37:49,486 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=80ad337678f5ab9b11d51704f5980d4a, daughterA=5868b317f196d2a5e51ca2d7baef0437, daughterB=e7b653549005a2a5c4cce0702ee0803c
2024-11-17T20:37:49,486 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=80ad337678f5ab9b11d51704f5980d4a, daughterA=5868b317f196d2a5e51ca2d7baef0437, daughterB=e7b653549005a2a5c4cce0702ee0803c
2024-11-17T20:37:49,486 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=80ad337678f5ab9b11d51704f5980d4a, daughterA=5868b317f196d2a5e51ca2d7baef0437, daughterB=e7b653549005a2a5c4cce0702ee0803c
2024-11-17T20:37:49,495 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=80ad337678f5ab9b11d51704f5980d4a, UNASSIGN}]
2024-11-17T20:37:49,497 INFO  [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=80ad337678f5ab9b11d51704f5980d4a, UNASSIGN
2024-11-17T20:37:49,498 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=80ad337678f5ab9b11d51704f5980d4a, regionState=CLOSING, regionLocation=0b0b0ae8de03,44889,1731875854003
2024-11-17T20:37:49,500 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=80ad337678f5ab9b11d51704f5980d4a, UNASSIGN because future has completed
2024-11-17T20:37:49,501 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: true: evictOnSplit: true: evictOnClose: false
2024-11-17T20:37:49,501 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; CloseRegionProcedure 80ad337678f5ab9b11d51704f5980d4a, server=0b0b0ae8de03,44889,1731875854003}]
2024-11-17T20:37:49,658 INFO  [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(122): Close 80ad337678f5ab9b11d51704f5980d4a
2024-11-17T20:37:49,658 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(136): Unassign region: split region: true: evictCache: true
2024-11-17T20:37:49,659 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1722): Closing 80ad337678f5ab9b11d51704f5980d4a, disabling compactions & flushes
2024-11-17T20:37:49,659 INFO  [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1731875855142.80ad337678f5ab9b11d51704f5980d4a.
2024-11-17T20:37:49,659 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1731875855142.80ad337678f5ab9b11d51704f5980d4a.
2024-11-17T20:37:49,659 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1731875855142.80ad337678f5ab9b11d51704f5980d4a. after waiting 0 ms
2024-11-17T20:37:49,659 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1731875855142.80ad337678f5ab9b11d51704f5980d4a.
2024-11-17T20:37:49,659 INFO  [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(2902): Flushing 80ad337678f5ab9b11d51704f5980d4a 1/1 column families, dataSize=4.20 KB heapSize=4.75 KB
2024-11-17T20:37:49,665 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/.tmp/info/350de4acacd841a99c7e1020274ac7eb is 1080, key is row0093/info:/1731875869421/Put/seqid=0
2024-11-17T20:37:49,670 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44593 is added to blk_1073741848_1024 (size=9270)
2024-11-17T20:37:49,671 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741848_1024 (size=9270)
2024-11-17T20:37:49,672 INFO  [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.20 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/.tmp/info/350de4acacd841a99c7e1020274ac7eb
2024-11-17T20:37:49,719 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/.tmp/info/350de4acacd841a99c7e1020274ac7eb as hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/350de4acacd841a99c7e1020274ac7eb
2024-11-17T20:37:49,727 INFO  [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/350de4acacd841a99c7e1020274ac7eb, entries=4, sequenceid=127, filesize=9.1 K
2024-11-17T20:37:49,729 INFO  [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(3140): Finished flush of dataSize ~4.20 KB/4304, heapSize ~4.73 KB/4848, currentSize=0 B/0 for 80ad337678f5ab9b11d51704f5980d4a in 70ms, sequenceid=127, compaction requested=true
2024-11-17T20:37:49,731 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731875855142.80ad337678f5ab9b11d51704f5980d4a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/41d6987b2c5a4a219fadc9cb5d625183, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/0f005012635d42e3908daa1d9ac8589e, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/6f4eb6543f0d462b8d5ba06354f5e6a9, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/5ae1da2e369f493292bd376e3a11154e, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/9a397c9afce54727a86d220269ddc63e, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/aa9508865dcf4e518503a2a045bb09c1, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/48a8ae3e483c4fdb8da246b88d967783, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/7a32ae70e9014f1aab838b4bfd775e6f, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/7ed4d2f96d2f42d48d77257feaf87f64] to archive
2024-11-17T20:37:49,732 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731875855142.80ad337678f5ab9b11d51704f5980d4a.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-17T20:37:49,734 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731875855142.80ad337678f5ab9b11d51704f5980d4a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/41d6987b2c5a4a219fadc9cb5d625183 to hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/archive/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/41d6987b2c5a4a219fadc9cb5d625183
2024-11-17T20:37:49,735 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731875855142.80ad337678f5ab9b11d51704f5980d4a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/0f005012635d42e3908daa1d9ac8589e to hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/archive/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/0f005012635d42e3908daa1d9ac8589e
2024-11-17T20:37:49,736 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731875855142.80ad337678f5ab9b11d51704f5980d4a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/6f4eb6543f0d462b8d5ba06354f5e6a9 to hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/archive/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/6f4eb6543f0d462b8d5ba06354f5e6a9
2024-11-17T20:37:49,737 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731875855142.80ad337678f5ab9b11d51704f5980d4a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/5ae1da2e369f493292bd376e3a11154e to hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/archive/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/5ae1da2e369f493292bd376e3a11154e
2024-11-17T20:37:49,737 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731875855142.80ad337678f5ab9b11d51704f5980d4a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/9a397c9afce54727a86d220269ddc63e to hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/archive/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/9a397c9afce54727a86d220269ddc63e
2024-11-17T20:37:49,738 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731875855142.80ad337678f5ab9b11d51704f5980d4a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/aa9508865dcf4e518503a2a045bb09c1 to hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/archive/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/aa9508865dcf4e518503a2a045bb09c1
2024-11-17T20:37:49,739 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731875855142.80ad337678f5ab9b11d51704f5980d4a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/48a8ae3e483c4fdb8da246b88d967783 to hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/archive/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/48a8ae3e483c4fdb8da246b88d967783
2024-11-17T20:37:49,740 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731875855142.80ad337678f5ab9b11d51704f5980d4a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/7a32ae70e9014f1aab838b4bfd775e6f to hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/archive/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/7a32ae70e9014f1aab838b4bfd775e6f
2024-11-17T20:37:49,741 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731875855142.80ad337678f5ab9b11d51704f5980d4a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/7ed4d2f96d2f42d48d77257feaf87f64 to hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/archive/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/7ed4d2f96d2f42d48d77257feaf87f64
2024-11-17T20:37:49,746 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/recovered.edits/130.seqid, newMaxSeqId=130, maxSeqId=1
2024-11-17T20:37:49,747 INFO  [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1731875855142.80ad337678f5ab9b11d51704f5980d4a.
2024-11-17T20:37:49,747 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1676): Region close journal for 80ad337678f5ab9b11d51704f5980d4a:
Waiting for close lock at 1731875869659Running coprocessor pre-close hooks at 1731875869659Disabling compacts and flushes for region at 1731875869659Disabling writes for close at 1731875869659Obtaining lock to block concurrent updates at 1731875869659Preparing flush snapshotting stores in 80ad337678f5ab9b11d51704f5980d4a at 1731875869659Finished memstore snapshotting TestLogRolling-testLogRolling,,1731875855142.80ad337678f5ab9b11d51704f5980d4a., syncing WAL and waiting on mvcc, flushsize=dataSize=4304, getHeapSize=4848, getOffHeapSize=0, getCellsCount=4 at 1731875869660 (+1 ms)Flushing stores of TestLogRolling-testLogRolling,,1731875855142.80ad337678f5ab9b11d51704f5980d4a. at 1731875869660Flushing 80ad337678f5ab9b11d51704f5980d4a/info: creating writer at 1731875869660Flushing 80ad337678f5ab9b11d51704f5980d4a/info: appending metadata at 1731875869664 (+4 ms)Flushing 80ad337678f5ab9b11d51704f5980d4a/info: closing flushed file at 1731875869664Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@46024f5e: reopening flushed file at 1731875869717 (+53 ms)Finished flush of dataSize ~4.20 KB/4304, heapSize ~4.73 KB/4848, currentSize=0 B/0 for 80ad337678f5ab9b11d51704f5980d4a in 70ms, sequenceid=127, compaction requested=true at 1731875869729 (+12 ms)Writing region close event to WAL at 1731875869743 (+14 ms)Running coprocessor post-close hooks at 1731875869747 (+4 ms)Closed at 1731875869747
2024-11-17T20:37:49,749 INFO  [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(157): Closed 80ad337678f5ab9b11d51704f5980d4a
2024-11-17T20:37:49,750 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=80ad337678f5ab9b11d51704f5980d4a, regionState=CLOSED
2024-11-17T20:37:49,751 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; CloseRegionProcedure 80ad337678f5ab9b11d51704f5980d4a, server=0b0b0ae8de03,44889,1731875854003 because future has completed
2024-11-17T20:37:49,755 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8
2024-11-17T20:37:49,755 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; CloseRegionProcedure 80ad337678f5ab9b11d51704f5980d4a, server=0b0b0ae8de03,44889,1731875854003 in 252 msec
2024-11-17T20:37:49,757 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7
2024-11-17T20:37:49,757 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=80ad337678f5ab9b11d51704f5980d4a, UNASSIGN in 260 msec
2024-11-17T20:37:49,765 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:37:49,769 INFO  [PEWorker-4 {}] assignment.SplitTableRegionProcedure(728): pid=7 splitting 3 storefiles, region=80ad337678f5ab9b11d51704f5980d4a, threads=3
2024-11-17T20:37:49,771 DEBUG [StoreFileSplitter-pool-1 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/350de4acacd841a99c7e1020274ac7eb for region: 80ad337678f5ab9b11d51704f5980d4a
2024-11-17T20:37:49,771 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/15b37024d5e44bb39aa4456aabc889a6 for region: 80ad337678f5ab9b11d51704f5980d4a
2024-11-17T20:37:49,771 DEBUG [StoreFileSplitter-pool-2 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/7a56768371864672b2334aa7f10cf37b for region: 80ad337678f5ab9b11d51704f5980d4a
2024-11-17T20:37:49,782 DEBUG [StoreFileSplitter-pool-0 {}] regionserver.HRegionFileSystem(650): Will create HFileLink file for hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/15b37024d5e44bb39aa4456aabc889a6, top=true
2024-11-17T20:37:49,782 DEBUG [StoreFileSplitter-pool-1 {}] regionserver.HRegionFileSystem(650): Will create HFileLink file for hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/350de4acacd841a99c7e1020274ac7eb, top=true
2024-11-17T20:37:49,791 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741849_1025 (size=27)
2024-11-17T20:37:49,791 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44593 is added to blk_1073741849_1025 (size=27)
2024-11-17T20:37:49,791 INFO  [StoreFileSplitter-pool-1 {}] regionserver.HRegionFileSystem(691): Created linkFile:hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/TestLogRolling-testLogRolling=80ad337678f5ab9b11d51704f5980d4a-350de4acacd841a99c7e1020274ac7eb for child: e7b653549005a2a5c4cce0702ee0803c, parent: 80ad337678f5ab9b11d51704f5980d4a
2024-11-17T20:37:49,792 INFO  [StoreFileSplitter-pool-0 {}] regionserver.HRegionFileSystem(691): Created linkFile:hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/TestLogRolling-testLogRolling=80ad337678f5ab9b11d51704f5980d4a-15b37024d5e44bb39aa4456aabc889a6 for child: e7b653549005a2a5c4cce0702ee0803c, parent: 80ad337678f5ab9b11d51704f5980d4a
2024-11-17T20:37:49,792 DEBUG [StoreFileSplitter-pool-1 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/350de4acacd841a99c7e1020274ac7eb for region: 80ad337678f5ab9b11d51704f5980d4a
2024-11-17T20:37:49,792 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/15b37024d5e44bb39aa4456aabc889a6 for region: 80ad337678f5ab9b11d51704f5980d4a
2024-11-17T20:37:49,799 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44593 is added to blk_1073741850_1026 (size=27)
2024-11-17T20:37:49,800 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741850_1026 (size=27)
2024-11-17T20:37:49,800 DEBUG [StoreFileSplitter-pool-2 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/7a56768371864672b2334aa7f10cf37b for region: 80ad337678f5ab9b11d51704f5980d4a
2024-11-17T20:37:49,802 DEBUG [PEWorker-4 {}] assignment.SplitTableRegionProcedure(802): pid=7 split storefiles for region 80ad337678f5ab9b11d51704f5980d4a Daughter A: [hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/5868b317f196d2a5e51ca2d7baef0437/info/7a56768371864672b2334aa7f10cf37b.80ad337678f5ab9b11d51704f5980d4a] storefiles, Daughter B: [hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/TestLogRolling-testLogRolling=80ad337678f5ab9b11d51704f5980d4a-15b37024d5e44bb39aa4456aabc889a6, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/TestLogRolling-testLogRolling=80ad337678f5ab9b11d51704f5980d4a-350de4acacd841a99c7e1020274ac7eb, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/7a56768371864672b2334aa7f10cf37b.80ad337678f5ab9b11d51704f5980d4a] storefiles.
2024-11-17T20:37:49,816 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44593 is added to blk_1073741851_1027 (size=71)
2024-11-17T20:37:49,817 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741851_1027 (size=71)
2024-11-17T20:37:49,835 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:49,842 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:50,005 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:50,214 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:37:50,228 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44593 is added to blk_1073741852_1028 (size=71)
2024-11-17T20:37:50,228 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741852_1028 (size=71)
2024-11-17T20:37:50,230 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:37:50,243 DEBUG [PEWorker-4 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/5868b317f196d2a5e51ca2d7baef0437/recovered.edits/130.seqid, newMaxSeqId=130, maxSeqId=-1
2024-11-17T20:37:50,248 DEBUG [PEWorker-4 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/recovered.edits/130.seqid, newMaxSeqId=130, maxSeqId=-1
2024-11-17T20:37:50,251 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1731875855142.80ad337678f5ab9b11d51704f5980d4a.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1731875870250"},{"qualifier":"splitA","vlen":70,"tag":[],"timestamp":"1731875870250"},{"qualifier":"splitB","vlen":70,"tag":[],"timestamp":"1731875870250"}]},"ts":"1731875870250"}
2024-11-17T20:37:50,251 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1731875869480.5868b317f196d2a5e51ca2d7baef0437.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1731875870250"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731875870250"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1731875870250"}]},"ts":"1731875870250"}
2024-11-17T20:37:50,251 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,row0062,1731875869480.e7b653549005a2a5c4cce0702ee0803c.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1731875870250"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731875870250"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1731875870250"}]},"ts":"1731875870250"}
2024-11-17T20:37:50,275 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=5868b317f196d2a5e51ca2d7baef0437, ASSIGN}, {pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=e7b653549005a2a5c4cce0702ee0803c, ASSIGN}]
2024-11-17T20:37:50,277 INFO  [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=e7b653549005a2a5c4cce0702ee0803c, ASSIGN
2024-11-17T20:37:50,277 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=5868b317f196d2a5e51ca2d7baef0437, ASSIGN
2024-11-17T20:37:50,278 INFO  [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=e7b653549005a2a5c4cce0702ee0803c, ASSIGN; state=SPLITTING_NEW, location=0b0b0ae8de03,44889,1731875854003; forceNewPlan=false, retain=false
2024-11-17T20:37:50,279 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=5868b317f196d2a5e51ca2d7baef0437, ASSIGN; state=SPLITTING_NEW, location=0b0b0ae8de03,44889,1731875854003; forceNewPlan=false, retain=false
2024-11-17T20:37:50,429 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=5868b317f196d2a5e51ca2d7baef0437, regionState=OPENING, regionLocation=0b0b0ae8de03,44889,1731875854003
2024-11-17T20:37:50,429 INFO  [PEWorker-2 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=e7b653549005a2a5c4cce0702ee0803c, regionState=OPENING, regionLocation=0b0b0ae8de03,44889,1731875854003
2024-11-17T20:37:50,431 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=5868b317f196d2a5e51ca2d7baef0437, ASSIGN because future has completed
2024-11-17T20:37:50,431 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5868b317f196d2a5e51ca2d7baef0437, server=0b0b0ae8de03,44889,1731875854003}]
2024-11-17T20:37:50,432 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=e7b653549005a2a5c4cce0702ee0803c, ASSIGN because future has completed
2024-11-17T20:37:50,433 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=13, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure e7b653549005a2a5c4cce0702ee0803c, server=0b0b0ae8de03,44889,1731875854003}]
2024-11-17T20:37:50,587 INFO  [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,,1731875869480.5868b317f196d2a5e51ca2d7baef0437.
2024-11-17T20:37:50,587 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 5868b317f196d2a5e51ca2d7baef0437, NAME => 'TestLogRolling-testLogRolling,,1731875869480.5868b317f196d2a5e51ca2d7baef0437.', STARTKEY => '', ENDKEY => 'row0062'}
2024-11-17T20:37:50,587 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling 5868b317f196d2a5e51ca2d7baef0437
2024-11-17T20:37:50,587 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1731875869480.5868b317f196d2a5e51ca2d7baef0437.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T20:37:50,588 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 5868b317f196d2a5e51ca2d7baef0437
2024-11-17T20:37:50,588 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 5868b317f196d2a5e51ca2d7baef0437
2024-11-17T20:37:50,589 INFO  [StoreOpener-5868b317f196d2a5e51ca2d7baef0437-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 5868b317f196d2a5e51ca2d7baef0437 
2024-11-17T20:37:50,590 INFO  [StoreOpener-5868b317f196d2a5e51ca2d7baef0437-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5868b317f196d2a5e51ca2d7baef0437 columnFamilyName info
2024-11-17T20:37:50,590 DEBUG [StoreOpener-5868b317f196d2a5e51ca2d7baef0437-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:37:50,602 DEBUG [StoreOpener-5868b317f196d2a5e51ca2d7baef0437-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/5868b317f196d2a5e51ca2d7baef0437/info/7a56768371864672b2334aa7f10cf37b.80ad337678f5ab9b11d51704f5980d4a->hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/7a56768371864672b2334aa7f10cf37b-bottom
2024-11-17T20:37:50,603 INFO  [StoreOpener-5868b317f196d2a5e51ca2d7baef0437-1 {}] regionserver.HStore(327): Store=5868b317f196d2a5e51ca2d7baef0437/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-17T20:37:50,603 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 5868b317f196d2a5e51ca2d7baef0437
2024-11-17T20:37:50,604 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/5868b317f196d2a5e51ca2d7baef0437
2024-11-17T20:37:50,605 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/5868b317f196d2a5e51ca2d7baef0437
2024-11-17T20:37:50,605 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 5868b317f196d2a5e51ca2d7baef0437
2024-11-17T20:37:50,605 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 5868b317f196d2a5e51ca2d7baef0437
2024-11-17T20:37:50,607 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 5868b317f196d2a5e51ca2d7baef0437
2024-11-17T20:37:50,608 INFO  [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 5868b317f196d2a5e51ca2d7baef0437; next sequenceid=131; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=791518, jitterRate=0.00646798312664032}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-17T20:37:50,608 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5868b317f196d2a5e51ca2d7baef0437
2024-11-17T20:37:50,608 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 5868b317f196d2a5e51ca2d7baef0437:
Running coprocessor pre-open hook at 1731875870588Writing region info on filesystem at 1731875870588Initializing all the Stores at 1731875870589 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731875870589Cleaning up temporary data from old regions at 1731875870605 (+16 ms)Running coprocessor post-open hooks at 1731875870608 (+3 ms)Region opened successfully at 1731875870608
2024-11-17T20:37:50,609 INFO  [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,,1731875869480.5868b317f196d2a5e51ca2d7baef0437., pid=12, masterSystemTime=1731875870583
2024-11-17T20:37:50,609 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.CompactSplit(403): Add compact mark for store 5868b317f196d2a5e51ca2d7baef0437:info, priority=-2147483648, current under compaction store size is 1
2024-11-17T20:37:50,610 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking
2024-11-17T20:37:50,610 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-17T20:37:50,610 INFO  [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.HStore(1527): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,,1731875869480.5868b317f196d2a5e51ca2d7baef0437.
2024-11-17T20:37:50,610 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.HStore(1541): 5868b317f196d2a5e51ca2d7baef0437/info is initiating minor compaction (all files)
2024-11-17T20:37:50,610 INFO  [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5868b317f196d2a5e51ca2d7baef0437/info in TestLogRolling-testLogRolling,,1731875869480.5868b317f196d2a5e51ca2d7baef0437.
2024-11-17T20:37:50,611 INFO  [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/5868b317f196d2a5e51ca2d7baef0437/info/7a56768371864672b2334aa7f10cf37b.80ad337678f5ab9b11d51704f5980d4a->hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/7a56768371864672b2334aa7f10cf37b-bottom] into tmpdir=hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/5868b317f196d2a5e51ca2d7baef0437/.tmp, totalSize=85.3 K
2024-11-17T20:37:50,611 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a56768371864672b2334aa7f10cf37b.80ad337678f5ab9b11d51704f5980d4a, keycount=38, bloomtype=ROW, size=85.3 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731875865209
2024-11-17T20:37:50,612 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,,1731875869480.5868b317f196d2a5e51ca2d7baef0437.
2024-11-17T20:37:50,612 INFO  [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,,1731875869480.5868b317f196d2a5e51ca2d7baef0437.
2024-11-17T20:37:50,612 INFO  [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,row0062,1731875869480.e7b653549005a2a5c4cce0702ee0803c.
2024-11-17T20:37:50,612 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7752): Opening region: {ENCODED => e7b653549005a2a5c4cce0702ee0803c, NAME => 'TestLogRolling-testLogRolling,row0062,1731875869480.e7b653549005a2a5c4cce0702ee0803c.', STARTKEY => 'row0062', ENDKEY => ''}
2024-11-17T20:37:50,612 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling e7b653549005a2a5c4cce0702ee0803c
2024-11-17T20:37:50,612 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,row0062,1731875869480.e7b653549005a2a5c4cce0702ee0803c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T20:37:50,613 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=5868b317f196d2a5e51ca2d7baef0437, regionState=OPEN, openSeqNum=131, regionLocation=0b0b0ae8de03,44889,1731875854003
2024-11-17T20:37:50,613 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7794): checking encryption for e7b653549005a2a5c4cce0702ee0803c
2024-11-17T20:37:50,613 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7797): checking classloading for e7b653549005a2a5c4cce0702ee0803c
2024-11-17T20:37:50,614 INFO  [StoreOpener-e7b653549005a2a5c4cce0702ee0803c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region e7b653549005a2a5c4cce0702ee0803c 
2024-11-17T20:37:50,614 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44889 {}] regionserver.HRegion(8855): Flush requested on 1588230740
2024-11-17T20:37:50,614 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all.
2024-11-17T20:37:50,614 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=5.11 KB heapSize=8.96 KB
2024-11-17T20:37:50,614 INFO  [StoreOpener-e7b653549005a2a5c4cce0702ee0803c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e7b653549005a2a5c4cce0702ee0803c columnFamilyName info
2024-11-17T20:37:50,614 DEBUG [StoreOpener-e7b653549005a2a5c4cce0702ee0803c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:37:50,615 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5868b317f196d2a5e51ca2d7baef0437, server=0b0b0ae8de03,44889,1731875854003 because future has completed
2024-11-17T20:37:50,621 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10
2024-11-17T20:37:50,621 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 5868b317f196d2a5e51ca2d7baef0437, server=0b0b0ae8de03,44889,1731875854003 in 185 msec
2024-11-17T20:37:50,623 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=5868b317f196d2a5e51ca2d7baef0437, ASSIGN in 346 msec
2024-11-17T20:37:50,628 DEBUG [StoreOpener-e7b653549005a2a5c4cce0702ee0803c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/7a56768371864672b2334aa7f10cf37b.80ad337678f5ab9b11d51704f5980d4a->hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/7a56768371864672b2334aa7f10cf37b-top
2024-11-17T20:37:50,632 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/hbase/meta/1588230740/.tmp/info/e242a2c2ade340fdb4b5f8beb3722f11 is 193, key is TestLogRolling-testLogRolling,row0062,1731875869480.e7b653549005a2a5c4cce0702ee0803c./info:regioninfo/1731875870429/Put/seqid=0
2024-11-17T20:37:50,633 DEBUG [StoreOpener-e7b653549005a2a5c4cce0702ee0803c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/TestLogRolling-testLogRolling=80ad337678f5ab9b11d51704f5980d4a-15b37024d5e44bb39aa4456aabc889a6
2024-11-17T20:37:50,635 INFO  [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5868b317f196d2a5e51ca2d7baef0437#info#compaction#68 average throughput is 20.87 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-17T20:37:50,636 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/5868b317f196d2a5e51ca2d7baef0437/.tmp/info/af58242552bd4c988266741fab56c3fb is 1080, key is row0001/info:/1731875865209/Put/seqid=0
2024-11-17T20:37:50,637 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741853_1029 (size=9847)
2024-11-17T20:37:50,637 DEBUG [StoreOpener-e7b653549005a2a5c4cce0702ee0803c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/TestLogRolling-testLogRolling=80ad337678f5ab9b11d51704f5980d4a-350de4acacd841a99c7e1020274ac7eb
2024-11-17T20:37:50,637 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44593 is added to blk_1073741853_1029 (size=9847)
2024-11-17T20:37:50,637 INFO  [StoreOpener-e7b653549005a2a5c4cce0702ee0803c-1 {}] regionserver.HStore(327): Store=e7b653549005a2a5c4cce0702ee0803c/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-17T20:37:50,638 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.92 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/hbase/meta/1588230740/.tmp/info/e242a2c2ade340fdb4b5f8beb3722f11
2024-11-17T20:37:50,640 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1038): replaying wal for e7b653549005a2a5c4cce0702ee0803c
2024-11-17T20:37:50,641 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c
2024-11-17T20:37:50,642 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741854_1030 (size=70862)
2024-11-17T20:37:50,642 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44593 is added to blk_1073741854_1030 (size=70862)
2024-11-17T20:37:50,643 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c
2024-11-17T20:37:50,643 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1048): stopping wal replay for e7b653549005a2a5c4cce0702ee0803c
2024-11-17T20:37:50,643 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1060): Cleaning up temporary data for e7b653549005a2a5c4cce0702ee0803c
2024-11-17T20:37:50,645 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1093): writing seq id for e7b653549005a2a5c4cce0702ee0803c
2024-11-17T20:37:50,646 INFO  [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1114): Opened e7b653549005a2a5c4cce0702ee0803c; next sequenceid=131; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=819216, jitterRate=0.041688233613967896}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-17T20:37:50,646 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e7b653549005a2a5c4cce0702ee0803c
2024-11-17T20:37:50,646 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1006): Region open journal for e7b653549005a2a5c4cce0702ee0803c:
Running coprocessor pre-open hook at 1731875870613Writing region info on filesystem at 1731875870613Initializing all the Stores at 1731875870613Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731875870613Cleaning up temporary data from old regions at 1731875870643 (+30 ms)Running coprocessor post-open hooks at 1731875870646 (+3 ms)Region opened successfully at 1731875870646
2024-11-17T20:37:50,647 INFO  [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,row0062,1731875869480.e7b653549005a2a5c4cce0702ee0803c., pid=13, masterSystemTime=1731875870583
2024-11-17T20:37:50,647 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.CompactSplit(403): Add compact mark for store e7b653549005a2a5c4cce0702ee0803c:info, priority=-2147483648, current under compaction store size is 2
2024-11-17T20:37:50,647 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-17T20:37:50,647 DEBUG [RS:0;0b0b0ae8de03:44889-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-17T20:37:50,648 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/5868b317f196d2a5e51ca2d7baef0437/.tmp/info/af58242552bd4c988266741fab56c3fb as hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/5868b317f196d2a5e51ca2d7baef0437/info/af58242552bd4c988266741fab56c3fb
2024-11-17T20:37:50,648 INFO  [RS:0;0b0b0ae8de03:44889-longCompactions-0 {}] regionserver.HStore(1527): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,row0062,1731875869480.e7b653549005a2a5c4cce0702ee0803c.
2024-11-17T20:37:50,648 DEBUG [RS:0;0b0b0ae8de03:44889-longCompactions-0 {}] regionserver.HStore(1541): e7b653549005a2a5c4cce0702ee0803c/info is initiating minor compaction (all files)
2024-11-17T20:37:50,648 INFO  [RS:0;0b0b0ae8de03:44889-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7b653549005a2a5c4cce0702ee0803c/info in TestLogRolling-testLogRolling,row0062,1731875869480.e7b653549005a2a5c4cce0702ee0803c.
2024-11-17T20:37:50,648 INFO  [RS:0;0b0b0ae8de03:44889-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/7a56768371864672b2334aa7f10cf37b.80ad337678f5ab9b11d51704f5980d4a->hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/7a56768371864672b2334aa7f10cf37b-top, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/TestLogRolling-testLogRolling=80ad337678f5ab9b11d51704f5980d4a-15b37024d5e44bb39aa4456aabc889a6, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/TestLogRolling-testLogRolling=80ad337678f5ab9b11d51704f5980d4a-350de4acacd841a99c7e1020274ac7eb] into tmpdir=hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/.tmp, totalSize=116.0 K
2024-11-17T20:37:50,649 DEBUG [RS:0;0b0b0ae8de03:44889-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a56768371864672b2334aa7f10cf37b.80ad337678f5ab9b11d51704f5980d4a, keycount=38, bloomtype=ROW, size=85.3 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1731875865209
2024-11-17T20:37:50,649 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,row0062,1731875869480.e7b653549005a2a5c4cce0702ee0803c.
2024-11-17T20:37:50,649 INFO  [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,row0062,1731875869480.e7b653549005a2a5c4cce0702ee0803c.
2024-11-17T20:37:50,649 DEBUG [RS:0;0b0b0ae8de03:44889-longCompactions-0 {}] compactions.Compactor(225): Compacting TestLogRolling-testLogRolling=80ad337678f5ab9b11d51704f5980d4a-15b37024d5e44bb39aa4456aabc889a6, keycount=16, bloomtype=ROW, size=21.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731875869385
2024-11-17T20:37:50,650 DEBUG [RS:0;0b0b0ae8de03:44889-longCompactions-0 {}] compactions.Compactor(225): Compacting TestLogRolling-testLogRolling=80ad337678f5ab9b11d51704f5980d4a-350de4acacd841a99c7e1020274ac7eb, keycount=4, bloomtype=ROW, size=9.1 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731875869421
2024-11-17T20:37:50,650 INFO  [PEWorker-4 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=e7b653549005a2a5c4cce0702ee0803c, regionState=OPEN, openSeqNum=131, regionLocation=0b0b0ae8de03,44889,1731875854003
2024-11-17T20:37:50,652 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure e7b653549005a2a5c4cce0702ee0803c, server=0b0b0ae8de03,44889,1731875854003 because future has completed
2024-11-17T20:37:50,654 INFO  [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 1 (all) file(s) in 5868b317f196d2a5e51ca2d7baef0437/info of 5868b317f196d2a5e51ca2d7baef0437 into af58242552bd4c988266741fab56c3fb(size=69.2 K), total size for store is 69.2 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-17T20:37:50,654 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5868b317f196d2a5e51ca2d7baef0437:

2024-11-17T20:37:50,654 INFO  [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1731875869480.5868b317f196d2a5e51ca2d7baef0437., storeName=5868b317f196d2a5e51ca2d7baef0437/info, priority=15, startTime=1731875870609; duration=0sec
2024-11-17T20:37:50,654 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-17T20:37:50,654 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5868b317f196d2a5e51ca2d7baef0437:info
2024-11-17T20:37:50,655 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=13, resume processing ppid=11
2024-11-17T20:37:50,655 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure e7b653549005a2a5c4cce0702ee0803c, server=0b0b0ae8de03,44889,1731875854003 in 221 msec
2024-11-17T20:37:50,658 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=7
2024-11-17T20:37:50,658 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=e7b653549005a2a5c4cce0702ee0803c, ASSIGN in 380 msec
2024-11-17T20:37:50,659 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=80ad337678f5ab9b11d51704f5980d4a, daughterA=5868b317f196d2a5e51ca2d7baef0437, daughterB=e7b653549005a2a5c4cce0702ee0803c in 1.1780 sec
2024-11-17T20:37:50,663 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/hbase/meta/1588230740/.tmp/ns/54fe64db21954794946b8f8d4c9bcdb7 is 43, key is default/ns:d/1731875855081/Put/seqid=0
2024-11-17T20:37:50,669 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44593 is added to blk_1073741855_1031 (size=5153)
2024-11-17T20:37:50,669 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741855_1031 (size=5153)
2024-11-17T20:37:50,669 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/hbase/meta/1588230740/.tmp/ns/54fe64db21954794946b8f8d4c9bcdb7
2024-11-17T20:37:50,673 INFO  [RS:0;0b0b0ae8de03:44889-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7b653549005a2a5c4cce0702ee0803c#info#compaction#70 average throughput is 17.96 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-17T20:37:50,674 DEBUG [RS:0;0b0b0ae8de03:44889-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/.tmp/info/c9de6c0321404977a0651bacf522e56b is 1080, key is row0062/info:/1731875867347/Put/seqid=0
2024-11-17T20:37:50,678 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741856_1032 (size=42984)
2024-11-17T20:37:50,679 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44593 is added to blk_1073741856_1032 (size=42984)
2024-11-17T20:37:50,685 DEBUG [RS:0;0b0b0ae8de03:44889-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/.tmp/info/c9de6c0321404977a0651bacf522e56b as hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/c9de6c0321404977a0651bacf522e56b
2024-11-17T20:37:50,691 INFO  [RS:0;0b0b0ae8de03:44889-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7b653549005a2a5c4cce0702ee0803c/info of e7b653549005a2a5c4cce0702ee0803c into c9de6c0321404977a0651bacf522e56b(size=42.0 K), total size for store is 42.0 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-17T20:37:50,692 DEBUG [RS:0;0b0b0ae8de03:44889-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7b653549005a2a5c4cce0702ee0803c:

2024-11-17T20:37:50,692 INFO  [RS:0;0b0b0ae8de03:44889-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731875869480.e7b653549005a2a5c4cce0702ee0803c., storeName=e7b653549005a2a5c4cce0702ee0803c/info, priority=13, startTime=1731875870647; duration=0sec
2024-11-17T20:37:50,692 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/hbase/meta/1588230740/.tmp/table/175957aec6a3414b988529672d9e9e4d is 65, key is TestLogRolling-testLogRolling/table:state/1731875855522/Put/seqid=0
2024-11-17T20:37:50,692 DEBUG [RS:0;0b0b0ae8de03:44889-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-17T20:37:50,692 DEBUG [RS:0;0b0b0ae8de03:44889-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7b653549005a2a5c4cce0702ee0803c:info
2024-11-17T20:37:50,696 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741857_1033 (size=5340)
2024-11-17T20:37:50,696 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44593 is added to blk_1073741857_1033 (size=5340)
2024-11-17T20:37:50,697 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=122 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/hbase/meta/1588230740/.tmp/table/175957aec6a3414b988529672d9e9e4d
2024-11-17T20:37:50,702 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/hbase/meta/1588230740/.tmp/info/e242a2c2ade340fdb4b5f8beb3722f11 as hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/hbase/meta/1588230740/info/e242a2c2ade340fdb4b5f8beb3722f11
2024-11-17T20:37:50,707 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/hbase/meta/1588230740/info/e242a2c2ade340fdb4b5f8beb3722f11, entries=30, sequenceid=17, filesize=9.6 K
2024-11-17T20:37:50,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/hbase/meta/1588230740/.tmp/ns/54fe64db21954794946b8f8d4c9bcdb7 as hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/hbase/meta/1588230740/ns/54fe64db21954794946b8f8d4c9bcdb7
2024-11-17T20:37:50,712 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/hbase/meta/1588230740/ns/54fe64db21954794946b8f8d4c9bcdb7, entries=2, sequenceid=17, filesize=5.0 K
2024-11-17T20:37:50,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/hbase/meta/1588230740/.tmp/table/175957aec6a3414b988529672d9e9e4d as hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/hbase/meta/1588230740/table/175957aec6a3414b988529672d9e9e4d
2024-11-17T20:37:50,718 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/hbase/meta/1588230740/table/175957aec6a3414b988529672d9e9e4d, entries=2, sequenceid=17, filesize=5.2 K
2024-11-17T20:37:50,719 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~5.11 KB/5234, heapSize ~8.66 KB/8872, currentSize=705 B/705 for 1588230740 in 105ms, sequenceid=17, compaction requested=false
2024-11-17T20:37:50,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1588230740:

2024-11-17T20:37:50,836 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:50,844 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:51,006 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:51,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:56198 deadline: 1731875881432, exception=org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1731875855142.80ad337678f5ab9b11d51704f5980d4a. is not online on 0b0b0ae8de03,44889,1731875854003
2024-11-17T20:37:51,464 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestLogRolling-testLogRolling,,1731875855142.80ad337678f5ab9b11d51704f5980d4a., hostname=0b0b0ae8de03,44889,1731875854003, seqNum=2 , the old value is region=TestLogRolling-testLogRolling,,1731875855142.80ad337678f5ab9b11d51704f5980d4a., hostname=0b0b0ae8de03,44889,1731875854003, seqNum=2, error=org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1731875855142.80ad337678f5ab9b11d51704f5980d4a. is not online on 0b0b0ae8de03,44889,1731875854003
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186)
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2943)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-11-17T20:37:51,465 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestLogRolling-testLogRolling,,1731875855142.80ad337678f5ab9b11d51704f5980d4a., hostname=0b0b0ae8de03,44889,1731875854003, seqNum=2 is org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1731875855142.80ad337678f5ab9b11d51704f5980d4a. is not online on 0b0b0ae8de03,44889,1731875854003
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186)
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2943)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-11-17T20:37:51,465 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=TestLogRolling-testLogRolling,,1731875855142.80ad337678f5ab9b11d51704f5980d4a., hostname=0b0b0ae8de03,44889,1731875854003, seqNum=2 from cache
2024-11-17T20:37:51,837 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:51,844 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:52,006 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:52,837 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:52,845 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:53,007 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:53,838 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:53,845 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:54,008 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:54,748 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:37:54,748 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:37:54,749 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:37:54,749 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:37:54,749 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:37:54,750 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:37:54,777 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:37:54,777 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:37:54,777 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:37:54,777 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:37:54,777 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:37:54,777 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:37:54,780 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:37:54,781 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:37:54,781 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:37:54,783 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:37:54,839 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:54,845 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:55,008 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:55,289 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-17T20:37:55,290 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:37:55,290 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:37:55,290 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:37:55,291 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:37:55,291 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:37:55,291 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:37:55,316 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:37:55,317 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:37:55,317 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:37:55,317 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:37:55,317 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:37:55,317 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:37:55,320 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:37:55,321 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:37:55,321 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:37:55,323 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:37:55,839 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:55,846 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:56,009 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:56,840 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:56,846 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:57,010 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:57,841 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:57,847 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:58,011 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:58,842 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:58,848 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:59,012 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:59,843 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:37:59,849 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:38:00,012 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:38:00,844 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:38:00,850 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:38:01,014 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:38:01,556 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRolling', row='row0097', locateType=CURRENT is [region=TestLogRolling-testLogRolling,row0062,1731875869480.e7b653549005a2a5c4cce0702ee0803c., hostname=0b0b0ae8de03,44889,1731875854003, seqNum=131]
2024-11-17T20:38:01,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] regionserver.HRegion(8855): Flush requested on e7b653549005a2a5c4cce0702ee0803c
2024-11-17T20:38:01,568 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7b653549005a2a5c4cce0702ee0803c 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-17T20:38:01,573 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/.tmp/info/b0a27013d6b5460d9c14d5c6542cac53 is 1080, key is row0097/info:/1731875881558/Put/seqid=0
2024-11-17T20:38:01,579 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44593 is added to blk_1073741858_1034 (size=12516)
2024-11-17T20:38:01,580 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741858_1034 (size=12516)
2024-11-17T20:38:01,583 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/.tmp/info/b0a27013d6b5460d9c14d5c6542cac53
2024-11-17T20:38:01,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/.tmp/info/b0a27013d6b5460d9c14d5c6542cac53 as hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/b0a27013d6b5460d9c14d5c6542cac53
2024-11-17T20:38:01,595 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/b0a27013d6b5460d9c14d5c6542cac53, entries=7, sequenceid=141, filesize=12.2 K
2024-11-17T20:38:01,596 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=12.61 KB/12912 for e7b653549005a2a5c4cce0702ee0803c in 28ms, sequenceid=141, compaction requested=false
2024-11-17T20:38:01,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7b653549005a2a5c4cce0702ee0803c:

2024-11-17T20:38:01,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] regionserver.HRegion(8855): Flush requested on e7b653549005a2a5c4cce0702ee0803c
2024-11-17T20:38:01,597 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7b653549005a2a5c4cce0702ee0803c 1/1 column families, dataSize=13.66 KB heapSize=14.88 KB
2024-11-17T20:38:01,601 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/.tmp/info/37d59245e5864773b65b39343477b3b5 is 1080, key is row0104/info:/1731875881569/Put/seqid=0
2024-11-17T20:38:01,605 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44593 is added to blk_1073741859_1035 (size=19000)
2024-11-17T20:38:01,605 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741859_1035 (size=19000)
2024-11-17T20:38:01,606 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.66 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/.tmp/info/37d59245e5864773b65b39343477b3b5
2024-11-17T20:38:01,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/.tmp/info/37d59245e5864773b65b39343477b3b5 as hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/37d59245e5864773b65b39343477b3b5
2024-11-17T20:38:01,617 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/37d59245e5864773b65b39343477b3b5, entries=13, sequenceid=157, filesize=18.6 K
2024-11-17T20:38:01,618 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~13.66 KB/13988, heapSize ~14.86 KB/15216, currentSize=10.51 KB/10760 for e7b653549005a2a5c4cce0702ee0803c in 21ms, sequenceid=157, compaction requested=true
2024-11-17T20:38:01,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7b653549005a2a5c4cce0702ee0803c:

2024-11-17T20:38:01,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7b653549005a2a5c4cce0702ee0803c:info, priority=-2147483648, current under compaction store size is 1
2024-11-17T20:38:01,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-17T20:38:01,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] regionserver.HRegion(8855): Flush requested on e7b653549005a2a5c4cce0702ee0803c
2024-11-17T20:38:01,619 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-17T20:38:01,619 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7b653549005a2a5c4cce0702ee0803c 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB
2024-11-17T20:38:01,620 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 74500 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-17T20:38:01,620 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.HStore(1541): e7b653549005a2a5c4cce0702ee0803c/info is initiating minor compaction (all files)
2024-11-17T20:38:01,620 INFO  [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7b653549005a2a5c4cce0702ee0803c/info in TestLogRolling-testLogRolling,row0062,1731875869480.e7b653549005a2a5c4cce0702ee0803c.
2024-11-17T20:38:01,620 INFO  [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/c9de6c0321404977a0651bacf522e56b, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/b0a27013d6b5460d9c14d5c6542cac53, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/37d59245e5864773b65b39343477b3b5] into tmpdir=hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/.tmp, totalSize=72.8 K
2024-11-17T20:38:01,621 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9de6c0321404977a0651bacf522e56b, keycount=35, bloomtype=ROW, size=42.0 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731875867347
2024-11-17T20:38:01,621 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0a27013d6b5460d9c14d5c6542cac53, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1731875881558
2024-11-17T20:38:01,621 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37d59245e5864773b65b39343477b3b5, keycount=13, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731875881569
2024-11-17T20:38:01,622 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/.tmp/info/f72450e12b26413ab2a93fd1c389a3e6 is 1080, key is row0117/info:/1731875881598/Put/seqid=0
2024-11-17T20:38:01,626 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741860_1036 (size=16828)
2024-11-17T20:38:01,627 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44593 is added to blk_1073741860_1036 (size=16828)
2024-11-17T20:38:01,627 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/.tmp/info/f72450e12b26413ab2a93fd1c389a3e6
2024-11-17T20:38:01,632 INFO  [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7b653549005a2a5c4cce0702ee0803c#info#compaction#75 average throughput is 28.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-17T20:38:01,632 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/.tmp/info/2c1b4c809ced4a9cb09e411171a1077b is 1080, key is row0062/info:/1731875867347/Put/seqid=0
2024-11-17T20:38:01,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/.tmp/info/f72450e12b26413ab2a93fd1c389a3e6 as hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/f72450e12b26413ab2a93fd1c389a3e6
2024-11-17T20:38:01,637 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44593 is added to blk_1073741861_1037 (size=64714)
2024-11-17T20:38:01,637 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741861_1037 (size=64714)
2024-11-17T20:38:01,638 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/f72450e12b26413ab2a93fd1c389a3e6, entries=11, sequenceid=171, filesize=16.4 K
2024-11-17T20:38:01,638 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=1.05 KB/1076 for e7b653549005a2a5c4cce0702ee0803c in 19ms, sequenceid=171, compaction requested=false
2024-11-17T20:38:01,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7b653549005a2a5c4cce0702ee0803c:

2024-11-17T20:38:01,642 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/.tmp/info/2c1b4c809ced4a9cb09e411171a1077b as hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/2c1b4c809ced4a9cb09e411171a1077b
2024-11-17T20:38:01,647 INFO  [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7b653549005a2a5c4cce0702ee0803c/info of e7b653549005a2a5c4cce0702ee0803c into 2c1b4c809ced4a9cb09e411171a1077b(size=63.2 K), total size for store is 79.6 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-17T20:38:01,647 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7b653549005a2a5c4cce0702ee0803c:

2024-11-17T20:38:01,647 INFO  [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731875869480.e7b653549005a2a5c4cce0702ee0803c., storeName=e7b653549005a2a5c4cce0702ee0803c/info, priority=13, startTime=1731875881618; duration=0sec
2024-11-17T20:38:01,647 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-17T20:38:01,647 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7b653549005a2a5c4cce0702ee0803c:info
2024-11-17T20:38:01,845 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:38:01,851 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:38:02,015 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:38:02,846 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:38:02,851 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:38:03,016 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:38:03,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] regionserver.HRegion(8855): Flush requested on e7b653549005a2a5c4cce0702ee0803c
2024-11-17T20:38:03,635 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7b653549005a2a5c4cce0702ee0803c 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-17T20:38:03,640 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/.tmp/info/958c090cc6e442228a246fc11cbfb51e is 1080, key is row0128/info:/1731875881620/Put/seqid=0
2024-11-17T20:38:03,644 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44593 is added to blk_1073741862_1038 (size=12516)
2024-11-17T20:38:03,645 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741862_1038 (size=12516)
2024-11-17T20:38:03,645 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/.tmp/info/958c090cc6e442228a246fc11cbfb51e
2024-11-17T20:38:03,651 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/.tmp/info/958c090cc6e442228a246fc11cbfb51e as hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/958c090cc6e442228a246fc11cbfb51e
2024-11-17T20:38:03,658 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/958c090cc6e442228a246fc11cbfb51e, entries=7, sequenceid=182, filesize=12.2 K
2024-11-17T20:38:03,659 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=11.56 KB/11836 for e7b653549005a2a5c4cce0702ee0803c in 24ms, sequenceid=182, compaction requested=true
2024-11-17T20:38:03,659 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7b653549005a2a5c4cce0702ee0803c:

2024-11-17T20:38:03,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7b653549005a2a5c4cce0702ee0803c:info, priority=-2147483648, current under compaction store size is 1
2024-11-17T20:38:03,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-17T20:38:03,659 DEBUG [RS:0;0b0b0ae8de03:44889-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-17T20:38:03,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] regionserver.HRegion(8855): Flush requested on e7b653549005a2a5c4cce0702ee0803c
2024-11-17T20:38:03,660 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7b653549005a2a5c4cce0702ee0803c 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB
2024-11-17T20:38:03,660 DEBUG [RS:0;0b0b0ae8de03:44889-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 94058 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-17T20:38:03,660 DEBUG [RS:0;0b0b0ae8de03:44889-longCompactions-0 {}] regionserver.HStore(1541): e7b653549005a2a5c4cce0702ee0803c/info is initiating minor compaction (all files)
2024-11-17T20:38:03,660 INFO  [RS:0;0b0b0ae8de03:44889-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7b653549005a2a5c4cce0702ee0803c/info in TestLogRolling-testLogRolling,row0062,1731875869480.e7b653549005a2a5c4cce0702ee0803c.
2024-11-17T20:38:03,660 INFO  [RS:0;0b0b0ae8de03:44889-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/2c1b4c809ced4a9cb09e411171a1077b, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/f72450e12b26413ab2a93fd1c389a3e6, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/958c090cc6e442228a246fc11cbfb51e] into tmpdir=hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/.tmp, totalSize=91.9 K
2024-11-17T20:38:03,661 DEBUG [RS:0;0b0b0ae8de03:44889-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c1b4c809ced4a9cb09e411171a1077b, keycount=55, bloomtype=ROW, size=63.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731875867347
2024-11-17T20:38:03,661 DEBUG [RS:0;0b0b0ae8de03:44889-longCompactions-0 {}] compactions.Compactor(225): Compacting f72450e12b26413ab2a93fd1c389a3e6, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731875881598
2024-11-17T20:38:03,662 DEBUG [RS:0;0b0b0ae8de03:44889-longCompactions-0 {}] compactions.Compactor(225): Compacting 958c090cc6e442228a246fc11cbfb51e, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731875881620
2024-11-17T20:38:03,664 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/.tmp/info/d293a33bb540411a8e63b01b2a6c15dc is 1080, key is row0135/info:/1731875883637/Put/seqid=0
2024-11-17T20:38:03,669 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741863_1039 (size=17906)
2024-11-17T20:38:03,669 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44593 is added to blk_1073741863_1039 (size=17906)
2024-11-17T20:38:03,669 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/.tmp/info/d293a33bb540411a8e63b01b2a6c15dc
2024-11-17T20:38:03,673 INFO  [RS:0;0b0b0ae8de03:44889-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7b653549005a2a5c4cce0702ee0803c#info#compaction#78 average throughput is 37.45 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-17T20:38:03,674 DEBUG [RS:0;0b0b0ae8de03:44889-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/.tmp/info/f9fadb7a9f3448769ea0300c79884b13 is 1080, key is row0062/info:/1731875867347/Put/seqid=0
2024-11-17T20:38:03,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/.tmp/info/d293a33bb540411a8e63b01b2a6c15dc as hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/d293a33bb540411a8e63b01b2a6c15dc
2024-11-17T20:38:03,677 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741864_1040 (size=84293)
2024-11-17T20:38:03,678 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44593 is added to blk_1073741864_1040 (size=84293)
2024-11-17T20:38:03,681 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/d293a33bb540411a8e63b01b2a6c15dc, entries=12, sequenceid=197, filesize=17.5 K
2024-11-17T20:38:03,682 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=10.51 KB/10760 for e7b653549005a2a5c4cce0702ee0803c in 22ms, sequenceid=197, compaction requested=false
2024-11-17T20:38:03,682 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7b653549005a2a5c4cce0702ee0803c:

2024-11-17T20:38:03,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] regionserver.HRegion(8855): Flush requested on e7b653549005a2a5c4cce0702ee0803c
2024-11-17T20:38:03,683 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7b653549005a2a5c4cce0702ee0803c 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB
2024-11-17T20:38:03,683 DEBUG [RS:0;0b0b0ae8de03:44889-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/.tmp/info/f9fadb7a9f3448769ea0300c79884b13 as hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/f9fadb7a9f3448769ea0300c79884b13
2024-11-17T20:38:03,687 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/.tmp/info/7f739ba8afd64373a735c7c1c012a6d6 is 1080, key is row0147/info:/1731875883661/Put/seqid=0
2024-11-17T20:38:03,689 INFO  [RS:0;0b0b0ae8de03:44889-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7b653549005a2a5c4cce0702ee0803c/info of e7b653549005a2a5c4cce0702ee0803c into f9fadb7a9f3448769ea0300c79884b13(size=82.3 K), total size for store is 99.8 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-17T20:38:03,689 DEBUG [RS:0;0b0b0ae8de03:44889-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7b653549005a2a5c4cce0702ee0803c:

2024-11-17T20:38:03,689 INFO  [RS:0;0b0b0ae8de03:44889-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731875869480.e7b653549005a2a5c4cce0702ee0803c., storeName=e7b653549005a2a5c4cce0702ee0803c/info, priority=13, startTime=1731875883659; duration=0sec
2024-11-17T20:38:03,689 DEBUG [RS:0;0b0b0ae8de03:44889-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-17T20:38:03,689 DEBUG [RS:0;0b0b0ae8de03:44889-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7b653549005a2a5c4cce0702ee0803c:info
2024-11-17T20:38:03,692 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44593 is added to blk_1073741865_1041 (size=16828)
2024-11-17T20:38:03,692 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741865_1041 (size=16828)
2024-11-17T20:38:03,692 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/.tmp/info/7f739ba8afd64373a735c7c1c012a6d6
2024-11-17T20:38:03,697 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/.tmp/info/7f739ba8afd64373a735c7c1c012a6d6 as hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/7f739ba8afd64373a735c7c1c012a6d6
2024-11-17T20:38:03,700 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/7f739ba8afd64373a735c7c1c012a6d6, entries=11, sequenceid=211, filesize=16.4 K
2024-11-17T20:38:03,701 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=3.15 KB/3228 for e7b653549005a2a5c4cce0702ee0803c in 18ms, sequenceid=211, compaction requested=true
2024-11-17T20:38:03,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7b653549005a2a5c4cce0702ee0803c:

2024-11-17T20:38:03,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7b653549005a2a5c4cce0702ee0803c:info, priority=-2147483648, current under compaction store size is 1
2024-11-17T20:38:03,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-17T20:38:03,702 DEBUG [RS:0;0b0b0ae8de03:44889-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-17T20:38:03,702 DEBUG [RS:0;0b0b0ae8de03:44889-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 119027 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-17T20:38:03,703 DEBUG [RS:0;0b0b0ae8de03:44889-longCompactions-0 {}] regionserver.HStore(1541): e7b653549005a2a5c4cce0702ee0803c/info is initiating minor compaction (all files)
2024-11-17T20:38:03,703 INFO  [RS:0;0b0b0ae8de03:44889-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7b653549005a2a5c4cce0702ee0803c/info in TestLogRolling-testLogRolling,row0062,1731875869480.e7b653549005a2a5c4cce0702ee0803c.
2024-11-17T20:38:03,703 INFO  [RS:0;0b0b0ae8de03:44889-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/f9fadb7a9f3448769ea0300c79884b13, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/d293a33bb540411a8e63b01b2a6c15dc, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/7f739ba8afd64373a735c7c1c012a6d6] into tmpdir=hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/.tmp, totalSize=116.2 K
2024-11-17T20:38:03,703 DEBUG [RS:0;0b0b0ae8de03:44889-longCompactions-0 {}] compactions.Compactor(225): Compacting f9fadb7a9f3448769ea0300c79884b13, keycount=73, bloomtype=ROW, size=82.3 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731875867347
2024-11-17T20:38:03,703 DEBUG [RS:0;0b0b0ae8de03:44889-longCompactions-0 {}] compactions.Compactor(225): Compacting d293a33bb540411a8e63b01b2a6c15dc, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731875883637
2024-11-17T20:38:03,704 DEBUG [RS:0;0b0b0ae8de03:44889-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f739ba8afd64373a735c7c1c012a6d6, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731875883661
2024-11-17T20:38:03,716 INFO  [RS:0;0b0b0ae8de03:44889-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7b653549005a2a5c4cce0702ee0803c#info#compaction#80 average throughput is 32.84 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-17T20:38:03,717 DEBUG [RS:0;0b0b0ae8de03:44889-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/.tmp/info/c0c9176b7bf84dd0aae6809669b55695 is 1080, key is row0062/info:/1731875867347/Put/seqid=0
2024-11-17T20:38:03,719 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44593 is added to blk_1073741866_1042 (size=109197)
2024-11-17T20:38:03,719 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741866_1042 (size=109197)
2024-11-17T20:38:03,725 DEBUG [RS:0;0b0b0ae8de03:44889-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/.tmp/info/c0c9176b7bf84dd0aae6809669b55695 as hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/c0c9176b7bf84dd0aae6809669b55695
2024-11-17T20:38:03,731 INFO  [RS:0;0b0b0ae8de03:44889-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7b653549005a2a5c4cce0702ee0803c/info of e7b653549005a2a5c4cce0702ee0803c into c0c9176b7bf84dd0aae6809669b55695(size=106.6 K), total size for store is 106.6 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-17T20:38:03,731 DEBUG [RS:0;0b0b0ae8de03:44889-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7b653549005a2a5c4cce0702ee0803c:

2024-11-17T20:38:03,731 INFO  [RS:0;0b0b0ae8de03:44889-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731875869480.e7b653549005a2a5c4cce0702ee0803c., storeName=e7b653549005a2a5c4cce0702ee0803c/info, priority=13, startTime=1731875883701; duration=0sec
2024-11-17T20:38:03,731 DEBUG [RS:0;0b0b0ae8de03:44889-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-17T20:38:03,731 DEBUG [RS:0;0b0b0ae8de03:44889-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7b653549005a2a5c4cce0702ee0803c:info
2024-11-17T20:38:03,748 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-17T20:38:03,847 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:38:03,852 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:38:04,017 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:38:04,848 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:38:04,853 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:38:05,018 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:38:05,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] regionserver.HRegion(8855): Flush requested on e7b653549005a2a5c4cce0702ee0803c
2024-11-17T20:38:05,708 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7b653549005a2a5c4cce0702ee0803c 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-17T20:38:05,712 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/.tmp/info/572f535374134c4a92cd7a44c2ae8392 is 1080, key is row0158/info:/1731875883684/Put/seqid=0
2024-11-17T20:38:05,716 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44593 is added to blk_1073741867_1043 (size=12516)
2024-11-17T20:38:05,716 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741867_1043 (size=12516)
2024-11-17T20:38:05,717 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/.tmp/info/572f535374134c4a92cd7a44c2ae8392
2024-11-17T20:38:05,722 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/.tmp/info/572f535374134c4a92cd7a44c2ae8392 as hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/572f535374134c4a92cd7a44c2ae8392
2024-11-17T20:38:05,728 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/572f535374134c4a92cd7a44c2ae8392, entries=7, sequenceid=223, filesize=12.2 K
2024-11-17T20:38:05,729 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=10.51 KB/10760 for e7b653549005a2a5c4cce0702ee0803c in 20ms, sequenceid=223, compaction requested=false
2024-11-17T20:38:05,729 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7b653549005a2a5c4cce0702ee0803c:

2024-11-17T20:38:05,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] regionserver.HRegion(8855): Flush requested on e7b653549005a2a5c4cce0702ee0803c
2024-11-17T20:38:05,730 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7b653549005a2a5c4cce0702ee0803c 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB
2024-11-17T20:38:05,734 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/.tmp/info/a44aef04875c46c39bf77ff7b483ce59 is 1080, key is row0165/info:/1731875885709/Put/seqid=0
2024-11-17T20:38:05,740 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741868_1044 (size=16828)
2024-11-17T20:38:05,740 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44593 is added to blk_1073741868_1044 (size=16828)
2024-11-17T20:38:05,741 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/.tmp/info/a44aef04875c46c39bf77ff7b483ce59
2024-11-17T20:38:05,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/.tmp/info/a44aef04875c46c39bf77ff7b483ce59 as hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/a44aef04875c46c39bf77ff7b483ce59
2024-11-17T20:38:05,762 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/a44aef04875c46c39bf77ff7b483ce59, entries=11, sequenceid=237, filesize=16.4 K
2024-11-17T20:38:05,763 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=12.61 KB/12912 for e7b653549005a2a5c4cce0702ee0803c in 33ms, sequenceid=237, compaction requested=true
2024-11-17T20:38:05,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7b653549005a2a5c4cce0702ee0803c:

2024-11-17T20:38:05,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7b653549005a2a5c4cce0702ee0803c:info, priority=-2147483648, current under compaction store size is 1
2024-11-17T20:38:05,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-17T20:38:05,763 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-17T20:38:05,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] regionserver.HRegion(8855): Flush requested on e7b653549005a2a5c4cce0702ee0803c
2024-11-17T20:38:05,764 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7b653549005a2a5c4cce0702ee0803c 1/1 column families, dataSize=13.66 KB heapSize=14.88 KB
2024-11-17T20:38:05,764 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 138541 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-17T20:38:05,764 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.HStore(1541): e7b653549005a2a5c4cce0702ee0803c/info is initiating minor compaction (all files)
2024-11-17T20:38:05,764 INFO  [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7b653549005a2a5c4cce0702ee0803c/info in TestLogRolling-testLogRolling,row0062,1731875869480.e7b653549005a2a5c4cce0702ee0803c.
2024-11-17T20:38:05,765 INFO  [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/c0c9176b7bf84dd0aae6809669b55695, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/572f535374134c4a92cd7a44c2ae8392, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/a44aef04875c46c39bf77ff7b483ce59] into tmpdir=hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/.tmp, totalSize=135.3 K
2024-11-17T20:38:05,765 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0c9176b7bf84dd0aae6809669b55695, keycount=96, bloomtype=ROW, size=106.6 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731875867347
2024-11-17T20:38:05,766 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] compactions.Compactor(225): Compacting 572f535374134c4a92cd7a44c2ae8392, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731875883684
2024-11-17T20:38:05,766 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] compactions.Compactor(225): Compacting a44aef04875c46c39bf77ff7b483ce59, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731875885709
2024-11-17T20:38:05,768 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/.tmp/info/6c6ff18a448c4238bfee2db950469874 is 1080, key is row0176/info:/1731875885731/Put/seqid=0
2024-11-17T20:38:05,772 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741869_1045 (size=19000)
2024-11-17T20:38:05,772 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44593 is added to blk_1073741869_1045 (size=19000)
2024-11-17T20:38:05,773 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.66 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/.tmp/info/6c6ff18a448c4238bfee2db950469874
2024-11-17T20:38:05,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/.tmp/info/6c6ff18a448c4238bfee2db950469874 as hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/6c6ff18a448c4238bfee2db950469874
2024-11-17T20:38:05,779 INFO  [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7b653549005a2a5c4cce0702ee0803c#info#compaction#84 average throughput is 38.99 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-17T20:38:05,780 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/.tmp/info/ae5c10b350ea4c6cbc23c9bb72bb8e22 is 1080, key is row0062/info:/1731875867347/Put/seqid=0
2024-11-17T20:38:05,783 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/6c6ff18a448c4238bfee2db950469874, entries=13, sequenceid=253, filesize=18.6 K
2024-11-17T20:38:05,784 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~13.66 KB/13988, heapSize ~14.86 KB/15216, currentSize=4.20 KB/4304 for e7b653549005a2a5c4cce0702ee0803c in 21ms, sequenceid=253, compaction requested=false
2024-11-17T20:38:05,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7b653549005a2a5c4cce0702ee0803c:

2024-11-17T20:38:05,786 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741870_1046 (size=128835)
2024-11-17T20:38:05,786 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44593 is added to blk_1073741870_1046 (size=128835)
2024-11-17T20:38:05,791 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/.tmp/info/ae5c10b350ea4c6cbc23c9bb72bb8e22 as hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/ae5c10b350ea4c6cbc23c9bb72bb8e22
2024-11-17T20:38:05,796 INFO  [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7b653549005a2a5c4cce0702ee0803c/info of e7b653549005a2a5c4cce0702ee0803c into ae5c10b350ea4c6cbc23c9bb72bb8e22(size=125.8 K), total size for store is 144.4 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-17T20:38:05,796 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7b653549005a2a5c4cce0702ee0803c:

2024-11-17T20:38:05,796 INFO  [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731875869480.e7b653549005a2a5c4cce0702ee0803c., storeName=e7b653549005a2a5c4cce0702ee0803c/info, priority=13, startTime=1731875885763; duration=0sec
2024-11-17T20:38:05,797 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-17T20:38:05,797 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7b653549005a2a5c4cce0702ee0803c:info
2024-11-17T20:38:05,849 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:38:05,854 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:38:06,019 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:38:06,850 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:38:06,854 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:38:07,019 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:38:07,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] regionserver.HRegion(8855): Flush requested on e7b653549005a2a5c4cce0702ee0803c
2024-11-17T20:38:07,788 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7b653549005a2a5c4cce0702ee0803c 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-17T20:38:07,793 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/.tmp/info/7464601fba4c44f09e257021eb564b19 is 1080, key is row0189/info:/1731875885765/Put/seqid=0
2024-11-17T20:38:07,798 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741871_1047 (size=12522)
2024-11-17T20:38:07,799 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44593 is added to blk_1073741871_1047 (size=12522)
2024-11-17T20:38:07,799 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/.tmp/info/7464601fba4c44f09e257021eb564b19
2024-11-17T20:38:07,805 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/.tmp/info/7464601fba4c44f09e257021eb564b19 as hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/7464601fba4c44f09e257021eb564b19
2024-11-17T20:38:07,810 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/7464601fba4c44f09e257021eb564b19, entries=7, sequenceid=264, filesize=12.2 K
2024-11-17T20:38:07,811 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=10.51 KB/10760 for e7b653549005a2a5c4cce0702ee0803c in 23ms, sequenceid=264, compaction requested=true
2024-11-17T20:38:07,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7b653549005a2a5c4cce0702ee0803c:

2024-11-17T20:38:07,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7b653549005a2a5c4cce0702ee0803c:info, priority=-2147483648, current under compaction store size is 1
2024-11-17T20:38:07,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-17T20:38:07,811 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-17T20:38:07,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] regionserver.HRegion(8855): Flush requested on e7b653549005a2a5c4cce0702ee0803c
2024-11-17T20:38:07,812 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7b653549005a2a5c4cce0702ee0803c 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB
2024-11-17T20:38:07,812 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 160357 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-17T20:38:07,812 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.HStore(1541): e7b653549005a2a5c4cce0702ee0803c/info is initiating minor compaction (all files)
2024-11-17T20:38:07,812 INFO  [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7b653549005a2a5c4cce0702ee0803c/info in TestLogRolling-testLogRolling,row0062,1731875869480.e7b653549005a2a5c4cce0702ee0803c.
2024-11-17T20:38:07,812 INFO  [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/ae5c10b350ea4c6cbc23c9bb72bb8e22, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/6c6ff18a448c4238bfee2db950469874, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/7464601fba4c44f09e257021eb564b19] into tmpdir=hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/.tmp, totalSize=156.6 K
2024-11-17T20:38:07,813 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae5c10b350ea4c6cbc23c9bb72bb8e22, keycount=114, bloomtype=ROW, size=125.8 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731875867347
2024-11-17T20:38:07,813 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c6ff18a448c4238bfee2db950469874, keycount=13, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731875885731
2024-11-17T20:38:07,813 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7464601fba4c44f09e257021eb564b19, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731875885765
2024-11-17T20:38:07,815 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/.tmp/info/a5310950765149beb8a5db62d111de6e is 1080, key is row0196/info:/1731875887789/Put/seqid=0
2024-11-17T20:38:07,820 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44593 is added to blk_1073741872_1048 (size=16839)
2024-11-17T20:38:07,820 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741872_1048 (size=16839)
2024-11-17T20:38:07,821 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/.tmp/info/a5310950765149beb8a5db62d111de6e
2024-11-17T20:38:07,826 INFO  [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7b653549005a2a5c4cce0702ee0803c#info#compaction#87 average throughput is 45.83 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-17T20:38:07,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/.tmp/info/a5310950765149beb8a5db62d111de6e as hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/a5310950765149beb8a5db62d111de6e
2024-11-17T20:38:07,827 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/.tmp/info/575e8c9130554e82aeb78d907bd518d8 is 1080, key is row0062/info:/1731875867347/Put/seqid=0
2024-11-17T20:38:07,832 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44593 is added to blk_1073741873_1049 (size=150576)
2024-11-17T20:38:07,832 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741873_1049 (size=150576)
2024-11-17T20:38:07,833 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/a5310950765149beb8a5db62d111de6e, entries=11, sequenceid=278, filesize=16.4 K
2024-11-17T20:38:07,834 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=10.51 KB/10760 for e7b653549005a2a5c4cce0702ee0803c in 21ms, sequenceid=278, compaction requested=false
2024-11-17T20:38:07,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7b653549005a2a5c4cce0702ee0803c:

2024-11-17T20:38:07,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] regionserver.HRegion(8855): Flush requested on e7b653549005a2a5c4cce0702ee0803c
2024-11-17T20:38:07,836 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7b653549005a2a5c4cce0702ee0803c 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB
2024-11-17T20:38:07,838 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/.tmp/info/575e8c9130554e82aeb78d907bd518d8 as hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/575e8c9130554e82aeb78d907bd518d8
2024-11-17T20:38:07,840 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/.tmp/info/d7c5fb45d0e14c298d9b5745f79147c9 is 1080, key is row0207/info:/1731875887813/Put/seqid=0
2024-11-17T20:38:07,844 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44593 is added to blk_1073741874_1050 (size=16839)
2024-11-17T20:38:07,844 INFO  [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7b653549005a2a5c4cce0702ee0803c/info of e7b653549005a2a5c4cce0702ee0803c into 575e8c9130554e82aeb78d907bd518d8(size=147.0 K), total size for store is 163.5 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-17T20:38:07,844 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7b653549005a2a5c4cce0702ee0803c:

2024-11-17T20:38:07,844 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741874_1050 (size=16839)
2024-11-17T20:38:07,844 INFO  [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731875869480.e7b653549005a2a5c4cce0702ee0803c., storeName=e7b653549005a2a5c4cce0702ee0803c/info, priority=13, startTime=1731875887811; duration=0sec
2024-11-17T20:38:07,844 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-17T20:38:07,844 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7b653549005a2a5c4cce0702ee0803c:info
2024-11-17T20:38:07,844 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/.tmp/info/d7c5fb45d0e14c298d9b5745f79147c9
2024-11-17T20:38:07,850 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/.tmp/info/d7c5fb45d0e14c298d9b5745f79147c9 as hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/d7c5fb45d0e14c298d9b5745f79147c9
2024-11-17T20:38:07,851 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:38:07,854 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/d7c5fb45d0e14c298d9b5745f79147c9, entries=11, sequenceid=292, filesize=16.4 K
2024-11-17T20:38:07,855 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=7.36 KB/7532 for e7b653549005a2a5c4cce0702ee0803c in 19ms, sequenceid=292, compaction requested=true
2024-11-17T20:38:07,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7b653549005a2a5c4cce0702ee0803c:

2024-11-17T20:38:07,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7b653549005a2a5c4cce0702ee0803c:info, priority=-2147483648, current under compaction store size is 1
2024-11-17T20:38:07,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-17T20:38:07,855 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:38:07,855 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-17T20:38:07,856 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 184254 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-17T20:38:07,856 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.HStore(1541): e7b653549005a2a5c4cce0702ee0803c/info is initiating minor compaction (all files)
2024-11-17T20:38:07,856 INFO  [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7b653549005a2a5c4cce0702ee0803c/info in TestLogRolling-testLogRolling,row0062,1731875869480.e7b653549005a2a5c4cce0702ee0803c.
2024-11-17T20:38:07,856 INFO  [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/575e8c9130554e82aeb78d907bd518d8, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/a5310950765149beb8a5db62d111de6e, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/d7c5fb45d0e14c298d9b5745f79147c9] into tmpdir=hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/.tmp, totalSize=179.9 K
2024-11-17T20:38:07,857 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] compactions.Compactor(225): Compacting 575e8c9130554e82aeb78d907bd518d8, keycount=134, bloomtype=ROW, size=147.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731875867347
2024-11-17T20:38:07,857 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5310950765149beb8a5db62d111de6e, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731875887789
2024-11-17T20:38:07,857 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7c5fb45d0e14c298d9b5745f79147c9, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731875887813
2024-11-17T20:38:07,869 INFO  [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7b653549005a2a5c4cce0702ee0803c#info#compaction#89 average throughput is 53.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-17T20:38:07,870 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/.tmp/info/a45d68b056bc4db484990c0b4af6e656 is 1080, key is row0062/info:/1731875867347/Put/seqid=0
2024-11-17T20:38:07,872 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44593 is added to blk_1073741875_1051 (size=174424)
2024-11-17T20:38:07,873 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741875_1051 (size=174424)
2024-11-17T20:38:07,878 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/.tmp/info/a45d68b056bc4db484990c0b4af6e656 as hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/a45d68b056bc4db484990c0b4af6e656
2024-11-17T20:38:07,883 INFO  [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7b653549005a2a5c4cce0702ee0803c/info of e7b653549005a2a5c4cce0702ee0803c into a45d68b056bc4db484990c0b4af6e656(size=170.3 K), total size for store is 170.3 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-17T20:38:07,884 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7b653549005a2a5c4cce0702ee0803c:

2024-11-17T20:38:07,884 INFO  [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731875869480.e7b653549005a2a5c4cce0702ee0803c., storeName=e7b653549005a2a5c4cce0702ee0803c/info, priority=13, startTime=1731875887855; duration=0sec
2024-11-17T20:38:07,884 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-17T20:38:07,884 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7b653549005a2a5c4cce0702ee0803c:info
2024-11-17T20:38:08,020 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:38:08,852 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:38:08,856 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:38:09,021 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:38:09,853 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:38:09,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] regionserver.HRegion(8855): Flush requested on e7b653549005a2a5c4cce0702ee0803c
2024-11-17T20:38:09,857 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7b653549005a2a5c4cce0702ee0803c 1/1 column families, dataSize=8.41 KB heapSize=9.25 KB
2024-11-17T20:38:09,857 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:38:09,917 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/.tmp/info/15b4dc66e30d4ad88258fc0a874c4007 is 1080, key is row0218/info:/1731875887838/Put/seqid=0
2024-11-17T20:38:09,925 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44593 is added to blk_1073741876_1052 (size=13602)
2024-11-17T20:38:09,926 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741876_1052 (size=13602)
2024-11-17T20:38:09,926 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.41 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/.tmp/info/15b4dc66e30d4ad88258fc0a874c4007
2024-11-17T20:38:09,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/.tmp/info/15b4dc66e30d4ad88258fc0a874c4007 as hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/15b4dc66e30d4ad88258fc0a874c4007
2024-11-17T20:38:09,938 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/15b4dc66e30d4ad88258fc0a874c4007, entries=8, sequenceid=305, filesize=13.3 K
2024-11-17T20:38:09,939 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~8.41 KB/8608, heapSize ~9.23 KB/9456, currentSize=10.51 KB/10760 for e7b653549005a2a5c4cce0702ee0803c in 83ms, sequenceid=305, compaction requested=false
2024-11-17T20:38:09,939 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7b653549005a2a5c4cce0702ee0803c:

2024-11-17T20:38:09,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] regionserver.HRegion(8855): Flush requested on e7b653549005a2a5c4cce0702ee0803c
2024-11-17T20:38:09,941 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7b653549005a2a5c4cce0702ee0803c 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB
2024-11-17T20:38:09,945 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/.tmp/info/1095cf1e2679477cb1066bd5dac175fc is 1080, key is row0226/info:/1731875889858/Put/seqid=0
2024-11-17T20:38:09,949 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44593 is added to blk_1073741877_1053 (size=17918)
2024-11-17T20:38:09,949 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741877_1053 (size=17918)
2024-11-17T20:38:09,950 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/.tmp/info/1095cf1e2679477cb1066bd5dac175fc
2024-11-17T20:38:09,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/.tmp/info/1095cf1e2679477cb1066bd5dac175fc as hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/1095cf1e2679477cb1066bd5dac175fc
2024-11-17T20:38:09,961 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/1095cf1e2679477cb1066bd5dac175fc, entries=12, sequenceid=320, filesize=17.5 K
2024-11-17T20:38:09,962 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=10.51 KB/10760 for e7b653549005a2a5c4cce0702ee0803c in 21ms, sequenceid=320, compaction requested=true
2024-11-17T20:38:09,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7b653549005a2a5c4cce0702ee0803c:

2024-11-17T20:38:09,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7b653549005a2a5c4cce0702ee0803c:info, priority=-2147483648, current under compaction store size is 1
2024-11-17T20:38:09,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-17T20:38:09,962 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-17T20:38:09,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] regionserver.HRegion(8855): Flush requested on e7b653549005a2a5c4cce0702ee0803c
2024-11-17T20:38:09,962 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7b653549005a2a5c4cce0702ee0803c 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB
2024-11-17T20:38:09,963 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 205944 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-17T20:38:09,963 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.HStore(1541): e7b653549005a2a5c4cce0702ee0803c/info is initiating minor compaction (all files)
2024-11-17T20:38:09,963 INFO  [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7b653549005a2a5c4cce0702ee0803c/info in TestLogRolling-testLogRolling,row0062,1731875869480.e7b653549005a2a5c4cce0702ee0803c.
2024-11-17T20:38:09,963 INFO  [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/a45d68b056bc4db484990c0b4af6e656, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/15b4dc66e30d4ad88258fc0a874c4007, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/1095cf1e2679477cb1066bd5dac175fc] into tmpdir=hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/.tmp, totalSize=201.1 K
2024-11-17T20:38:09,964 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] compactions.Compactor(225): Compacting a45d68b056bc4db484990c0b4af6e656, keycount=156, bloomtype=ROW, size=170.3 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731875867347
2024-11-17T20:38:09,964 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15b4dc66e30d4ad88258fc0a874c4007, keycount=8, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1731875887838
2024-11-17T20:38:09,965 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1095cf1e2679477cb1066bd5dac175fc, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731875889858
2024-11-17T20:38:09,966 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/.tmp/info/7002fcbcb10b4a2dae0636d1a5ff672d is 1080, key is row0238/info:/1731875889942/Put/seqid=0
2024-11-17T20:38:09,970 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44593 is added to blk_1073741878_1054 (size=16839)
2024-11-17T20:38:09,971 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741878_1054 (size=16839)
2024-11-17T20:38:09,971 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/.tmp/info/7002fcbcb10b4a2dae0636d1a5ff672d
2024-11-17T20:38:09,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/.tmp/info/7002fcbcb10b4a2dae0636d1a5ff672d as hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/7002fcbcb10b4a2dae0636d1a5ff672d
2024-11-17T20:38:09,978 INFO  [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7b653549005a2a5c4cce0702ee0803c#info#compaction#93 average throughput is 45.15 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-17T20:38:09,978 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/.tmp/info/7acc400874444454a955a86556441372 is 1080, key is row0062/info:/1731875867347/Put/seqid=0
2024-11-17T20:38:09,980 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44593 is added to blk_1073741879_1055 (size=196110)
2024-11-17T20:38:09,980 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741879_1055 (size=196110)
2024-11-17T20:38:09,982 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/7002fcbcb10b4a2dae0636d1a5ff672d, entries=11, sequenceid=334, filesize=16.4 K
2024-11-17T20:38:09,983 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=8.41 KB/8608 for e7b653549005a2a5c4cce0702ee0803c in 21ms, sequenceid=334, compaction requested=false
2024-11-17T20:38:09,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7b653549005a2a5c4cce0702ee0803c:

2024-11-17T20:38:09,985 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/.tmp/info/7acc400874444454a955a86556441372 as hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/7acc400874444454a955a86556441372
2024-11-17T20:38:09,990 INFO  [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7b653549005a2a5c4cce0702ee0803c/info of e7b653549005a2a5c4cce0702ee0803c into 7acc400874444454a955a86556441372(size=191.5 K), total size for store is 208.0 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-17T20:38:09,990 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7b653549005a2a5c4cce0702ee0803c:

2024-11-17T20:38:09,990 INFO  [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731875869480.e7b653549005a2a5c4cce0702ee0803c., storeName=e7b653549005a2a5c4cce0702ee0803c/info, priority=13, startTime=1731875889962; duration=0sec
2024-11-17T20:38:09,990 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-17T20:38:09,990 DEBUG [RS:0;0b0b0ae8de03:44889-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7b653549005a2a5c4cce0702ee0803c:info
2024-11-17T20:38:10,022 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:38:10,854 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:38:10,858 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:38:11,022 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:38:11,855 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:38:11,859 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:38:11,979 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(285): after writing there are 0 log files
2024-11-17T20:38:11,980 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0b0b0ae8de03%2C44889%2C1731875854003.1731875891980
2024-11-17T20:38:11,991 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:38:11,991 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:38:11,991 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:38:11,991 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:38:11,992 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:38:11,992 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/WALs/0b0b0ae8de03,44889,1731875854003/0b0b0ae8de03%2C44889%2C1731875854003.1731875854646 with entries=320, filesize=311.00 KB; new WAL /user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/WALs/0b0b0ae8de03,44889,1731875854003/0b0b0ae8de03%2C44889%2C1731875854003.1731875891980
2024-11-17T20:38:11,994 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44593 is added to blk_1073741833_1009 (size=318475)
2024-11-17T20:38:11,995 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741833_1009 (size=318475)
2024-11-17T20:38:12,000 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37791:37791),(127.0.0.1/127.0.0.1:43885:43885)]
2024-11-17T20:38:12,003 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 5868b317f196d2a5e51ca2d7baef0437:

2024-11-17T20:38:12,003 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=705 B heapSize=2.05 KB
2024-11-17T20:38:12,008 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/hbase/meta/1588230740/.tmp/info/a4349d8f6db84fa7bccc60da4715978f is 193, key is TestLogRolling-testLogRolling,row0062,1731875869480.e7b653549005a2a5c4cce0702ee0803c./info:regioninfo/1731875870650/Put/seqid=0
2024-11-17T20:38:12,012 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44593 is added to blk_1073741881_1057 (size=6223)
2024-11-17T20:38:12,012 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741881_1057 (size=6223)
2024-11-17T20:38:12,013 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=705 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/hbase/meta/1588230740/.tmp/info/a4349d8f6db84fa7bccc60da4715978f
2024-11-17T20:38:12,017 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/hbase/meta/1588230740/.tmp/info/a4349d8f6db84fa7bccc60da4715978f as hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/hbase/meta/1588230740/info/a4349d8f6db84fa7bccc60da4715978f
2024-11-17T20:38:12,022 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/hbase/meta/1588230740/info/a4349d8f6db84fa7bccc60da4715978f, entries=5, sequenceid=21, filesize=6.1 K
2024-11-17T20:38:12,023 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~705 B/705, heapSize ~1.29 KB/1320, currentSize=0 B/0 for 1588230740 in 20ms, sequenceid=21, compaction requested=false
2024-11-17T20:38:12,023 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1588230740:

2024-11-17T20:38:12,023 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing e7b653549005a2a5c4cce0702ee0803c 1/1 column families, dataSize=8.41 KB heapSize=9.25 KB
2024-11-17T20:38:12,023 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:38:12,026 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/.tmp/info/b46f13ccfde4455dadddd29c76684310 is 1080, key is row0249/info:/1731875889963/Put/seqid=0
2024-11-17T20:38:12,033 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44593 is added to blk_1073741882_1058 (size=13602)
2024-11-17T20:38:12,034 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741882_1058 (size=13602)
2024-11-17T20:38:12,034 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.41 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/.tmp/info/b46f13ccfde4455dadddd29c76684310
2024-11-17T20:38:12,039 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/.tmp/info/b46f13ccfde4455dadddd29c76684310 as hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/b46f13ccfde4455dadddd29c76684310
2024-11-17T20:38:12,043 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/b46f13ccfde4455dadddd29c76684310, entries=8, sequenceid=346, filesize=13.3 K
2024-11-17T20:38:12,044 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~8.41 KB/8608, heapSize ~9.23 KB/9456, currentSize=0 B/0 for e7b653549005a2a5c4cce0702ee0803c in 21ms, sequenceid=346, compaction requested=true
2024-11-17T20:38:12,044 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for e7b653549005a2a5c4cce0702ee0803c:

2024-11-17T20:38:12,045 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0b0b0ae8de03%2C44889%2C1731875854003.1731875892044
2024-11-17T20:38:12,048 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:38:12,049 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:38:12,049 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:38:12,049 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:38:12,049 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:38:12,049 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/WALs/0b0b0ae8de03,44889,1731875854003/0b0b0ae8de03%2C44889%2C1731875854003.1731875891980 with entries=2, filesize=723 B; new WAL /user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/WALs/0b0b0ae8de03,44889,1731875854003/0b0b0ae8de03%2C44889%2C1731875854003.1731875892044
2024-11-17T20:38:12,050 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44593 is added to blk_1073741880_1056 (size=731)
2024-11-17T20:38:12,050 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741880_1056 (size=731)
2024-11-17T20:38:12,053 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/WALs/0b0b0ae8de03,44889,1731875854003/0b0b0ae8de03%2C44889%2C1731875854003.1731875854646 to hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/oldWALs/0b0b0ae8de03%2C44889%2C1731875854003.1731875854646
2024-11-17T20:38:12,053 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43885:43885),(127.0.0.1/127.0.0.1:37791:37791)]
2024-11-17T20:38:12,053 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1])
2024-11-17T20:38:12,053 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-17T20:38:12,054 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/WALs/0b0b0ae8de03,44889,1731875854003/0b0b0ae8de03%2C44889%2C1731875854003.1731875891980 to hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/oldWALs/0b0b0ae8de03%2C44889%2C1731875854003.1731875891980
2024-11-17T20:38:12,054 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-17T20:38:12,054 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-17T20:38:12,054 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T20:38:12,054 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T20:38:12,054 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-17T20:38:12,054 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-17T20:38:12,054 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=967075515, stopped=false
2024-11-17T20:38:12,054 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=0b0b0ae8de03,33755,1731875853766
2024-11-17T20:38:12,103 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33755-0x1014a8451610000, quorum=127.0.0.1:55719, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-17T20:38:12,103 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44889-0x1014a8451610001, quorum=127.0.0.1:55719, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-17T20:38:12,103 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33755-0x1014a8451610000, quorum=127.0.0.1:55719, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:38:12,103 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44889-0x1014a8451610001, quorum=127.0.0.1:55719, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:38:12,103 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-17T20:38:12,103 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-17T20:38:12,103 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-17T20:38:12,103 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T20:38:12,103 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44889-0x1014a8451610001, quorum=127.0.0.1:55719, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-17T20:38:12,103 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:33755-0x1014a8451610000, quorum=127.0.0.1:55719, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-17T20:38:12,104 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '0b0b0ae8de03,44889,1731875854003' *****
2024-11-17T20:38:12,104 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-17T20:38:12,104 INFO  [RS:0;0b0b0ae8de03:44889 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-17T20:38:12,104 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-17T20:38:12,104 INFO  [RS:0;0b0b0ae8de03:44889 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-17T20:38:12,104 INFO  [RS:0;0b0b0ae8de03:44889 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-17T20:38:12,104 INFO  [RS:0;0b0b0ae8de03:44889 {}] regionserver.HRegionServer(3091): Received CLOSE for 5868b317f196d2a5e51ca2d7baef0437
2024-11-17T20:38:12,104 INFO  [RS:0;0b0b0ae8de03:44889 {}] regionserver.HRegionServer(3091): Received CLOSE for e7b653549005a2a5c4cce0702ee0803c
2024-11-17T20:38:12,104 INFO  [RS:0;0b0b0ae8de03:44889 {}] regionserver.HRegionServer(959): stopping server 0b0b0ae8de03,44889,1731875854003
2024-11-17T20:38:12,105 INFO  [RS:0;0b0b0ae8de03:44889 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-17T20:38:12,105 INFO  [RS:0;0b0b0ae8de03:44889 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;0b0b0ae8de03:44889.
2024-11-17T20:38:12,105 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5868b317f196d2a5e51ca2d7baef0437, disabling compactions & flushes
2024-11-17T20:38:12,105 DEBUG [RS:0;0b0b0ae8de03:44889 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-17T20:38:12,105 INFO  [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1731875869480.5868b317f196d2a5e51ca2d7baef0437.
2024-11-17T20:38:12,105 DEBUG [RS:0;0b0b0ae8de03:44889 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T20:38:12,105 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1731875869480.5868b317f196d2a5e51ca2d7baef0437.
2024-11-17T20:38:12,105 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1731875869480.5868b317f196d2a5e51ca2d7baef0437. after waiting 0 ms
2024-11-17T20:38:12,105 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1731875869480.5868b317f196d2a5e51ca2d7baef0437.
2024-11-17T20:38:12,105 INFO  [RS:0;0b0b0ae8de03:44889 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-17T20:38:12,105 INFO  [RS:0;0b0b0ae8de03:44889 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-17T20:38:12,105 INFO  [RS:0;0b0b0ae8de03:44889 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-17T20:38:12,105 INFO  [RS:0;0b0b0ae8de03:44889 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-17T20:38:12,105 INFO  [RS:0;0b0b0ae8de03:44889 {}] regionserver.HRegionServer(1321): Waiting on 3 regions to close
2024-11-17T20:38:12,105 DEBUG [RS:0;0b0b0ae8de03:44889 {}] regionserver.HRegionServer(1325): Online Regions={5868b317f196d2a5e51ca2d7baef0437=TestLogRolling-testLogRolling,,1731875869480.5868b317f196d2a5e51ca2d7baef0437., 1588230740=hbase:meta,,1.1588230740, e7b653549005a2a5c4cce0702ee0803c=TestLogRolling-testLogRolling,row0062,1731875869480.e7b653549005a2a5c4cce0702ee0803c.}
2024-11-17T20:38:12,105 DEBUG [RS:0;0b0b0ae8de03:44889 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 5868b317f196d2a5e51ca2d7baef0437, e7b653549005a2a5c4cce0702ee0803c
2024-11-17T20:38:12,105 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-17T20:38:12,105 INFO  [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-17T20:38:12,105 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-17T20:38:12,105 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-17T20:38:12,105 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-17T20:38:12,105 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731875869480.5868b317f196d2a5e51ca2d7baef0437.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/5868b317f196d2a5e51ca2d7baef0437/info/7a56768371864672b2334aa7f10cf37b.80ad337678f5ab9b11d51704f5980d4a->hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/7a56768371864672b2334aa7f10cf37b-bottom] to archive
2024-11-17T20:38:12,106 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731875869480.5868b317f196d2a5e51ca2d7baef0437.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-17T20:38:12,108 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731875869480.5868b317f196d2a5e51ca2d7baef0437.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/5868b317f196d2a5e51ca2d7baef0437/info/7a56768371864672b2334aa7f10cf37b.80ad337678f5ab9b11d51704f5980d4a to hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/archive/data/default/TestLogRolling-testLogRolling/5868b317f196d2a5e51ca2d7baef0437/info/7a56768371864672b2334aa7f10cf37b.80ad337678f5ab9b11d51704f5980d4a
2024-11-17T20:38:12,108 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731875869480.5868b317f196d2a5e51ca2d7baef0437.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=0b0b0ae8de03:33755 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 16 more
2024-11-17T20:38:12,108 WARN  [StoreCloser-TestLogRolling-testLogRolling,,1731875869480.5868b317f196d2a5e51ca2d7baef0437.-1 {}] regionserver.HStore(2414): Failed to report archival of files: []
2024-11-17T20:38:12,109 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1
2024-11-17T20:38:12,110 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-17T20:38:12,110 INFO  [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-17T20:38:12,110 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731875892105Running coprocessor pre-close hooks at 1731875892105Disabling compacts and flushes for region at 1731875892105Disabling writes for close at 1731875892105Writing region close event to WAL at 1731875892106 (+1 ms)Running coprocessor post-close hooks at 1731875892110 (+4 ms)Closed at 1731875892110
2024-11-17T20:38:12,110 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-17T20:38:12,111 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/5868b317f196d2a5e51ca2d7baef0437/recovered.edits/135.seqid, newMaxSeqId=135, maxSeqId=130
2024-11-17T20:38:12,112 INFO  [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1731875869480.5868b317f196d2a5e51ca2d7baef0437.
2024-11-17T20:38:12,112 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5868b317f196d2a5e51ca2d7baef0437:
Waiting for close lock at 1731875892105Running coprocessor pre-close hooks at 1731875892105Disabling compacts and flushes for region at 1731875892105Disabling writes for close at 1731875892105Writing region close event to WAL at 1731875892109 (+4 ms)Running coprocessor post-close hooks at 1731875892112 (+3 ms)Closed at 1731875892112
2024-11-17T20:38:12,112 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,,1731875869480.5868b317f196d2a5e51ca2d7baef0437.
2024-11-17T20:38:12,112 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e7b653549005a2a5c4cce0702ee0803c, disabling compactions & flushes
2024-11-17T20:38:12,112 INFO  [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,row0062,1731875869480.e7b653549005a2a5c4cce0702ee0803c.
2024-11-17T20:38:12,112 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,row0062,1731875869480.e7b653549005a2a5c4cce0702ee0803c.
2024-11-17T20:38:12,112 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,row0062,1731875869480.e7b653549005a2a5c4cce0702ee0803c. after waiting 0 ms
2024-11-17T20:38:12,112 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,row0062,1731875869480.e7b653549005a2a5c4cce0702ee0803c.
2024-11-17T20:38:12,113 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731875869480.e7b653549005a2a5c4cce0702ee0803c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/7a56768371864672b2334aa7f10cf37b.80ad337678f5ab9b11d51704f5980d4a->hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/80ad337678f5ab9b11d51704f5980d4a/info/7a56768371864672b2334aa7f10cf37b-top, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/TestLogRolling-testLogRolling=80ad337678f5ab9b11d51704f5980d4a-15b37024d5e44bb39aa4456aabc889a6, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/c9de6c0321404977a0651bacf522e56b, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/TestLogRolling-testLogRolling=80ad337678f5ab9b11d51704f5980d4a-350de4acacd841a99c7e1020274ac7eb, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/b0a27013d6b5460d9c14d5c6542cac53, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/2c1b4c809ced4a9cb09e411171a1077b, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/37d59245e5864773b65b39343477b3b5, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/f72450e12b26413ab2a93fd1c389a3e6, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/f9fadb7a9f3448769ea0300c79884b13, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/958c090cc6e442228a246fc11cbfb51e, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/d293a33bb540411a8e63b01b2a6c15dc, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/c0c9176b7bf84dd0aae6809669b55695, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/7f739ba8afd64373a735c7c1c012a6d6, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/572f535374134c4a92cd7a44c2ae8392, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/ae5c10b350ea4c6cbc23c9bb72bb8e22, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/a44aef04875c46c39bf77ff7b483ce59, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/6c6ff18a448c4238bfee2db950469874, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/575e8c9130554e82aeb78d907bd518d8, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/7464601fba4c44f09e257021eb564b19, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/a5310950765149beb8a5db62d111de6e, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/a45d68b056bc4db484990c0b4af6e656, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/d7c5fb45d0e14c298d9b5745f79147c9, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/15b4dc66e30d4ad88258fc0a874c4007, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/1095cf1e2679477cb1066bd5dac175fc] to archive
2024-11-17T20:38:12,113 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731875869480.e7b653549005a2a5c4cce0702ee0803c.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-17T20:38:12,115 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731875869480.e7b653549005a2a5c4cce0702ee0803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/7a56768371864672b2334aa7f10cf37b.80ad337678f5ab9b11d51704f5980d4a to hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/archive/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/7a56768371864672b2334aa7f10cf37b.80ad337678f5ab9b11d51704f5980d4a
2024-11-17T20:38:12,116 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731875869480.e7b653549005a2a5c4cce0702ee0803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/TestLogRolling-testLogRolling=80ad337678f5ab9b11d51704f5980d4a-15b37024d5e44bb39aa4456aabc889a6 to hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/archive/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/TestLogRolling-testLogRolling=80ad337678f5ab9b11d51704f5980d4a-15b37024d5e44bb39aa4456aabc889a6
2024-11-17T20:38:12,117 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731875869480.e7b653549005a2a5c4cce0702ee0803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/c9de6c0321404977a0651bacf522e56b to hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/archive/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/c9de6c0321404977a0651bacf522e56b
2024-11-17T20:38:12,118 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731875869480.e7b653549005a2a5c4cce0702ee0803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/TestLogRolling-testLogRolling=80ad337678f5ab9b11d51704f5980d4a-350de4acacd841a99c7e1020274ac7eb to hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/archive/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/TestLogRolling-testLogRolling=80ad337678f5ab9b11d51704f5980d4a-350de4acacd841a99c7e1020274ac7eb
2024-11-17T20:38:12,119 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731875869480.e7b653549005a2a5c4cce0702ee0803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/b0a27013d6b5460d9c14d5c6542cac53 to hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/archive/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/b0a27013d6b5460d9c14d5c6542cac53
2024-11-17T20:38:12,120 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731875869480.e7b653549005a2a5c4cce0702ee0803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/2c1b4c809ced4a9cb09e411171a1077b to hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/archive/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/2c1b4c809ced4a9cb09e411171a1077b
2024-11-17T20:38:12,121 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731875869480.e7b653549005a2a5c4cce0702ee0803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/37d59245e5864773b65b39343477b3b5 to hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/archive/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/37d59245e5864773b65b39343477b3b5
2024-11-17T20:38:12,122 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731875869480.e7b653549005a2a5c4cce0702ee0803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/f72450e12b26413ab2a93fd1c389a3e6 to hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/archive/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/f72450e12b26413ab2a93fd1c389a3e6
2024-11-17T20:38:12,122 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731875869480.e7b653549005a2a5c4cce0702ee0803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/f9fadb7a9f3448769ea0300c79884b13 to hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/archive/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/f9fadb7a9f3448769ea0300c79884b13
2024-11-17T20:38:12,123 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731875869480.e7b653549005a2a5c4cce0702ee0803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/958c090cc6e442228a246fc11cbfb51e to hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/archive/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/958c090cc6e442228a246fc11cbfb51e
2024-11-17T20:38:12,124 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731875869480.e7b653549005a2a5c4cce0702ee0803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/d293a33bb540411a8e63b01b2a6c15dc to hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/archive/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/d293a33bb540411a8e63b01b2a6c15dc
2024-11-17T20:38:12,125 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731875869480.e7b653549005a2a5c4cce0702ee0803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/c0c9176b7bf84dd0aae6809669b55695 to hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/archive/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/c0c9176b7bf84dd0aae6809669b55695
2024-11-17T20:38:12,126 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731875869480.e7b653549005a2a5c4cce0702ee0803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/7f739ba8afd64373a735c7c1c012a6d6 to hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/archive/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/7f739ba8afd64373a735c7c1c012a6d6
2024-11-17T20:38:12,127 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731875869480.e7b653549005a2a5c4cce0702ee0803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/572f535374134c4a92cd7a44c2ae8392 to hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/archive/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/572f535374134c4a92cd7a44c2ae8392
2024-11-17T20:38:12,127 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731875869480.e7b653549005a2a5c4cce0702ee0803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/ae5c10b350ea4c6cbc23c9bb72bb8e22 to hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/archive/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/ae5c10b350ea4c6cbc23c9bb72bb8e22
2024-11-17T20:38:12,128 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731875869480.e7b653549005a2a5c4cce0702ee0803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/a44aef04875c46c39bf77ff7b483ce59 to hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/archive/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/a44aef04875c46c39bf77ff7b483ce59
2024-11-17T20:38:12,129 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731875869480.e7b653549005a2a5c4cce0702ee0803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/6c6ff18a448c4238bfee2db950469874 to hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/archive/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/6c6ff18a448c4238bfee2db950469874
2024-11-17T20:38:12,130 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731875869480.e7b653549005a2a5c4cce0702ee0803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/575e8c9130554e82aeb78d907bd518d8 to hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/archive/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/575e8c9130554e82aeb78d907bd518d8
2024-11-17T20:38:12,131 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731875869480.e7b653549005a2a5c4cce0702ee0803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/7464601fba4c44f09e257021eb564b19 to hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/archive/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/7464601fba4c44f09e257021eb564b19
2024-11-17T20:38:12,132 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731875869480.e7b653549005a2a5c4cce0702ee0803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/a5310950765149beb8a5db62d111de6e to hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/archive/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/a5310950765149beb8a5db62d111de6e
2024-11-17T20:38:12,133 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731875869480.e7b653549005a2a5c4cce0702ee0803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/a45d68b056bc4db484990c0b4af6e656 to hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/archive/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/a45d68b056bc4db484990c0b4af6e656
2024-11-17T20:38:12,133 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731875869480.e7b653549005a2a5c4cce0702ee0803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/d7c5fb45d0e14c298d9b5745f79147c9 to hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/archive/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/d7c5fb45d0e14c298d9b5745f79147c9
2024-11-17T20:38:12,134 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731875869480.e7b653549005a2a5c4cce0702ee0803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/15b4dc66e30d4ad88258fc0a874c4007 to hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/archive/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/15b4dc66e30d4ad88258fc0a874c4007
2024-11-17T20:38:12,135 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731875869480.e7b653549005a2a5c4cce0702ee0803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/1095cf1e2679477cb1066bd5dac175fc to hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/archive/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/info/1095cf1e2679477cb1066bd5dac175fc
2024-11-17T20:38:12,135 WARN  [StoreCloser-TestLogRolling-testLogRolling,row0062,1731875869480.e7b653549005a2a5c4cce0702ee0803c.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [c9de6c0321404977a0651bacf522e56b=42984, b0a27013d6b5460d9c14d5c6542cac53=12516, 2c1b4c809ced4a9cb09e411171a1077b=64714, 37d59245e5864773b65b39343477b3b5=19000, f72450e12b26413ab2a93fd1c389a3e6=16828, f9fadb7a9f3448769ea0300c79884b13=84293, 958c090cc6e442228a246fc11cbfb51e=12516, d293a33bb540411a8e63b01b2a6c15dc=17906, c0c9176b7bf84dd0aae6809669b55695=109197, 7f739ba8afd64373a735c7c1c012a6d6=16828, 572f535374134c4a92cd7a44c2ae8392=12516, ae5c10b350ea4c6cbc23c9bb72bb8e22=128835, a44aef04875c46c39bf77ff7b483ce59=16828, 6c6ff18a448c4238bfee2db950469874=19000, 575e8c9130554e82aeb78d907bd518d8=150576, 7464601fba4c44f09e257021eb564b19=12522, a5310950765149beb8a5db62d111de6e=16839, a45d68b056bc4db484990c0b4af6e656=174424, d7c5fb45d0e14c298d9b5745f79147c9=16839, 15b4dc66e30d4ad88258fc0a874c4007=13602, 1095cf1e2679477cb1066bd5dac175fc=17918]
2024-11-17T20:38:12,138 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/data/default/TestLogRolling-testLogRolling/e7b653549005a2a5c4cce0702ee0803c/recovered.edits/349.seqid, newMaxSeqId=349, maxSeqId=130
2024-11-17T20:38:12,139 INFO  [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,row0062,1731875869480.e7b653549005a2a5c4cce0702ee0803c.
2024-11-17T20:38:12,139 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e7b653549005a2a5c4cce0702ee0803c:
Waiting for close lock at 1731875892112Running coprocessor pre-close hooks at 1731875892112Disabling compacts and flushes for region at 1731875892112Disabling writes for close at 1731875892112Writing region close event to WAL at 1731875892135 (+23 ms)Running coprocessor post-close hooks at 1731875892139 (+4 ms)Closed at 1731875892139
2024-11-17T20:38:12,139 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,row0062,1731875869480.e7b653549005a2a5c4cce0702ee0803c.
2024-11-17T20:38:12,305 INFO  [RS:0;0b0b0ae8de03:44889 {}] regionserver.HRegionServer(976): stopping server 0b0b0ae8de03,44889,1731875854003; all regions closed.
2024-11-17T20:38:12,306 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:38:12,307 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:38:12,307 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:38:12,307 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:38:12,307 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:38:12,311 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44593 is added to blk_1073741834_1010 (size=8107)
2024-11-17T20:38:12,311 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741834_1010 (size=8107)
2024-11-17T20:38:12,316 DEBUG [RS:0;0b0b0ae8de03:44889 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/oldWALs
2024-11-17T20:38:12,316 INFO  [RS:0;0b0b0ae8de03:44889 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 0b0b0ae8de03%2C44889%2C1731875854003.meta:.meta(num 1731875854987)
2024-11-17T20:38:12,317 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:38:12,317 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:38:12,317 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:38:12,317 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:38:12,317 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:38:12,319 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741883_1059 (size=780)
2024-11-17T20:38:12,319 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44593 is added to blk_1073741883_1059 (size=780)
2024-11-17T20:38:12,322 DEBUG [RS:0;0b0b0ae8de03:44889 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/oldWALs
2024-11-17T20:38:12,322 INFO  [RS:0;0b0b0ae8de03:44889 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 0b0b0ae8de03%2C44889%2C1731875854003:(num 1731875892044)
2024-11-17T20:38:12,322 DEBUG [RS:0;0b0b0ae8de03:44889 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T20:38:12,322 INFO  [RS:0;0b0b0ae8de03:44889 {}] regionserver.LeaseManager(133): Closed leases
2024-11-17T20:38:12,322 INFO  [RS:0;0b0b0ae8de03:44889 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-17T20:38:12,323 INFO  [RS:0;0b0b0ae8de03:44889 {}] hbase.ChoreService(370): Chore service for: regionserver/0b0b0ae8de03:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-17T20:38:12,323 INFO  [RS:0;0b0b0ae8de03:44889 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-17T20:38:12,323 INFO  [regionserver/0b0b0ae8de03:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-17T20:38:12,323 INFO  [RS:0;0b0b0ae8de03:44889 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44889
2024-11-17T20:38:12,372 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44889-0x1014a8451610001, quorum=127.0.0.1:55719, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/0b0b0ae8de03,44889,1731875854003
2024-11-17T20:38:12,372 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33755-0x1014a8451610000, quorum=127.0.0.1:55719, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-17T20:38:12,372 INFO  [RS:0;0b0b0ae8de03:44889 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-17T20:38:12,383 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [0b0b0ae8de03,44889,1731875854003]
2024-11-17T20:38:12,393 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/0b0b0ae8de03,44889,1731875854003 already deleted, retry=false
2024-11-17T20:38:12,393 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 0b0b0ae8de03,44889,1731875854003 expired; onlineServers=0
2024-11-17T20:38:12,393 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '0b0b0ae8de03,33755,1731875853766' *****
2024-11-17T20:38:12,394 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-17T20:38:12,394 INFO  [M:0;0b0b0ae8de03:33755 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-17T20:38:12,394 INFO  [M:0;0b0b0ae8de03:33755 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-17T20:38:12,394 DEBUG [M:0;0b0b0ae8de03:33755 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-17T20:38:12,395 DEBUG [M:0;0b0b0ae8de03:33755 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-17T20:38:12,395 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster-HFileCleaner.large.0-1731875854352 {}] cleaner.HFileCleaner(306): Exit Thread[master/0b0b0ae8de03:0:becomeActiveMaster-HFileCleaner.large.0-1731875854352,5,FailOnTimeoutGroup]
2024-11-17T20:38:12,395 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-17T20:38:12,395 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster-HFileCleaner.small.0-1731875854352 {}] cleaner.HFileCleaner(306): Exit Thread[master/0b0b0ae8de03:0:becomeActiveMaster-HFileCleaner.small.0-1731875854352,5,FailOnTimeoutGroup]
2024-11-17T20:38:12,395 INFO  [M:0;0b0b0ae8de03:33755 {}] hbase.ChoreService(370): Chore service for: master/0b0b0ae8de03:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-17T20:38:12,396 INFO  [M:0;0b0b0ae8de03:33755 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-17T20:38:12,396 DEBUG [M:0;0b0b0ae8de03:33755 {}] master.HMaster(1795): Stopping service threads
2024-11-17T20:38:12,396 INFO  [M:0;0b0b0ae8de03:33755 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-17T20:38:12,396 INFO  [M:0;0b0b0ae8de03:33755 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-17T20:38:12,396 INFO  [M:0;0b0b0ae8de03:33755 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-17T20:38:12,397 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-17T20:38:12,408 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33755-0x1014a8451610000, quorum=127.0.0.1:55719, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-17T20:38:12,408 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33755-0x1014a8451610000, quorum=127.0.0.1:55719, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:38:12,409 DEBUG [M:0;0b0b0ae8de03:33755 {}] zookeeper.ZKUtil(347): master:33755-0x1014a8451610000, quorum=127.0.0.1:55719, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-17T20:38:12,409 WARN  [M:0;0b0b0ae8de03:33755 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-17T20:38:12,410 INFO  [M:0;0b0b0ae8de03:33755 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/.lastflushedseqids
2024-11-17T20:38:12,421 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44593 is added to blk_1073741884_1060 (size=228)
2024-11-17T20:38:12,421 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741884_1060 (size=228)
2024-11-17T20:38:12,422 INFO  [M:0;0b0b0ae8de03:33755 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-17T20:38:12,422 INFO  [M:0;0b0b0ae8de03:33755 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-17T20:38:12,422 DEBUG [M:0;0b0b0ae8de03:33755 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-17T20:38:12,422 INFO  [M:0;0b0b0ae8de03:33755 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T20:38:12,422 DEBUG [M:0;0b0b0ae8de03:33755 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T20:38:12,422 DEBUG [M:0;0b0b0ae8de03:33755 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-17T20:38:12,422 DEBUG [M:0;0b0b0ae8de03:33755 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T20:38:12,423 INFO  [M:0;0b0b0ae8de03:33755 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=51.43 KB heapSize=63.38 KB
2024-11-17T20:38:12,438 DEBUG [M:0;0b0b0ae8de03:33755 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/67c82a8b15114450831a47d0a971a634 is 82, key is hbase:meta,,1/info:regioninfo/1731875855017/Put/seqid=0
2024-11-17T20:38:12,442 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741885_1061 (size=5672)
2024-11-17T20:38:12,442 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44593 is added to blk_1073741885_1061 (size=5672)
2024-11-17T20:38:12,442 INFO  [M:0;0b0b0ae8de03:33755 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/67c82a8b15114450831a47d0a971a634
2024-11-17T20:38:12,459 DEBUG [M:0;0b0b0ae8de03:33755 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/86744ae776574ce099877e4b507d083b is 750, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731875855528/Put/seqid=0
2024-11-17T20:38:12,463 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741886_1062 (size=7090)
2024-11-17T20:38:12,463 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44593 is added to blk_1073741886_1062 (size=7090)
2024-11-17T20:38:12,464 INFO  [M:0;0b0b0ae8de03:33755 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=50.83 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/86744ae776574ce099877e4b507d083b
2024-11-17T20:38:12,468 INFO  [M:0;0b0b0ae8de03:33755 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 86744ae776574ce099877e4b507d083b
2024-11-17T20:38:12,482 DEBUG [M:0;0b0b0ae8de03:33755 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/05ec424363f94e5f97c07d6c221f851f is 69, key is 0b0b0ae8de03,44889,1731875854003/rs:state/1731875854479/Put/seqid=0
2024-11-17T20:38:12,483 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44889-0x1014a8451610001, quorum=127.0.0.1:55719, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-17T20:38:12,483 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44889-0x1014a8451610001, quorum=127.0.0.1:55719, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-17T20:38:12,483 INFO  [RS:0;0b0b0ae8de03:44889 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-17T20:38:12,483 INFO  [RS:0;0b0b0ae8de03:44889 {}] regionserver.HRegionServer(1031): Exiting; stopping=0b0b0ae8de03,44889,1731875854003; zookeeper connection closed.
2024-11-17T20:38:12,483 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3da95482 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3da95482
2024-11-17T20:38:12,483 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-17T20:38:12,486 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741887_1063 (size=5156)
2024-11-17T20:38:12,486 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44593 is added to blk_1073741887_1063 (size=5156)
2024-11-17T20:38:12,486 INFO  [M:0;0b0b0ae8de03:33755 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/05ec424363f94e5f97c07d6c221f851f
2024-11-17T20:38:12,504 DEBUG [M:0;0b0b0ae8de03:33755 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/665f67f43a824214a71c1ed5fa843c3b is 52, key is load_balancer_on/state:d/1731875855138/Put/seqid=0
2024-11-17T20:38:12,508 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44593 is added to blk_1073741888_1064 (size=5056)
2024-11-17T20:38:12,509 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741888_1064 (size=5056)
2024-11-17T20:38:12,509 INFO  [M:0;0b0b0ae8de03:33755 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/665f67f43a824214a71c1ed5fa843c3b
2024-11-17T20:38:12,513 DEBUG [M:0;0b0b0ae8de03:33755 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/67c82a8b15114450831a47d0a971a634 as hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/67c82a8b15114450831a47d0a971a634
2024-11-17T20:38:12,515 INFO  [regionserver/0b0b0ae8de03:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-17T20:38:12,517 INFO  [M:0;0b0b0ae8de03:33755 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/67c82a8b15114450831a47d0a971a634, entries=8, sequenceid=125, filesize=5.5 K
2024-11-17T20:38:12,518 DEBUG [M:0;0b0b0ae8de03:33755 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/86744ae776574ce099877e4b507d083b as hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/86744ae776574ce099877e4b507d083b
2024-11-17T20:38:12,522 INFO  [M:0;0b0b0ae8de03:33755 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 86744ae776574ce099877e4b507d083b
2024-11-17T20:38:12,522 INFO  [M:0;0b0b0ae8de03:33755 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/86744ae776574ce099877e4b507d083b, entries=13, sequenceid=125, filesize=6.9 K
2024-11-17T20:38:12,523 DEBUG [M:0;0b0b0ae8de03:33755 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/05ec424363f94e5f97c07d6c221f851f as hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/05ec424363f94e5f97c07d6c221f851f
2024-11-17T20:38:12,527 INFO  [M:0;0b0b0ae8de03:33755 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/05ec424363f94e5f97c07d6c221f851f, entries=1, sequenceid=125, filesize=5.0 K
2024-11-17T20:38:12,528 DEBUG [M:0;0b0b0ae8de03:33755 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/665f67f43a824214a71c1ed5fa843c3b as hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/665f67f43a824214a71c1ed5fa843c3b
2024-11-17T20:38:12,532 INFO  [M:0;0b0b0ae8de03:33755 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35449/user/jenkins/test-data/952635c3-9207-bd30-2682-d478f7d0f61f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/665f67f43a824214a71c1ed5fa843c3b, entries=1, sequenceid=125, filesize=4.9 K
2024-11-17T20:38:12,533 INFO  [M:0;0b0b0ae8de03:33755 {}] regionserver.HRegion(3140): Finished flush of dataSize ~51.43 KB/52663, heapSize ~63.32 KB/64840, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 111ms, sequenceid=125, compaction requested=false
2024-11-17T20:38:12,534 INFO  [M:0;0b0b0ae8de03:33755 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T20:38:12,534 DEBUG [M:0;0b0b0ae8de03:33755 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731875892422Disabling compacts and flushes for region at 1731875892422Disabling writes for close at 1731875892422Obtaining lock to block concurrent updates at 1731875892423 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731875892423Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=52663, getHeapSize=64840, getOffHeapSize=0, getCellsCount=148 at 1731875892423Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731875892424 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731875892424Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731875892437 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731875892437Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731875892446 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731875892459 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731875892459Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731875892468 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731875892482 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731875892482Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731875892490 (+8 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731875892504 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731875892504Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@71bdde8f: reopening flushed file at 1731875892512 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@48cc4df0: reopening flushed file at 1731875892517 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@69b97faa: reopening flushed file at 1731875892522 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5d3533d: reopening flushed file at 1731875892527 (+5 ms)Finished flush of dataSize ~51.43 KB/52663, heapSize ~63.32 KB/64840, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 111ms, sequenceid=125, compaction requested=false at 1731875892533 (+6 ms)Writing region close event to WAL at 1731875892534 (+1 ms)Closed at 1731875892534
2024-11-17T20:38:12,535 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:38:12,535 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:38:12,535 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:38:12,535 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:38:12,535 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:38:12,536 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44593 is added to blk_1073741830_1006 (size=61332)
2024-11-17T20:38:12,537 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741830_1006 (size=61332)
2024-11-17T20:38:12,537 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-17T20:38:12,537 INFO  [M:0;0b0b0ae8de03:33755 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-17T20:38:12,537 INFO  [M:0;0b0b0ae8de03:33755 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33755
2024-11-17T20:38:12,537 INFO  [M:0;0b0b0ae8de03:33755 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-17T20:38:12,645 INFO  [M:0;0b0b0ae8de03:33755 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-17T20:38:12,645 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33755-0x1014a8451610000, quorum=127.0.0.1:55719, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-17T20:38:12,645 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33755-0x1014a8451610000, quorum=127.0.0.1:55719, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-17T20:38:12,651 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@42d059d{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T20:38:12,652 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@444decb3{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-17T20:38:12,652 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-17T20:38:12,652 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@e483a61{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-17T20:38:12,652 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6f4aa33e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1a3196ea-489f-0fa1-8ed8-e2e462983867/hadoop.log.dir/,STOPPED}
2024-11-17T20:38:12,654 WARN  [BP-909746748-172.17.0.2-1731875851585 heartbeating to localhost/127.0.0.1:35449 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-17T20:38:12,654 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-17T20:38:12,654 WARN  [BP-909746748-172.17.0.2-1731875851585 heartbeating to localhost/127.0.0.1:35449 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-909746748-172.17.0.2-1731875851585 (Datanode Uuid 503c1710-e801-46be-9e8c-b0894925d9a7) service to localhost/127.0.0.1:35449
2024-11-17T20:38:12,654 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-17T20:38:12,655 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1a3196ea-489f-0fa1-8ed8-e2e462983867/cluster_14a128d9-7999-521f-922f-c4fb1ce6abae/data/data3/current/BP-909746748-172.17.0.2-1731875851585 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T20:38:12,655 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1a3196ea-489f-0fa1-8ed8-e2e462983867/cluster_14a128d9-7999-521f-922f-c4fb1ce6abae/data/data4/current/BP-909746748-172.17.0.2-1731875851585 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T20:38:12,655 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-17T20:38:12,657 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@24bba582{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T20:38:12,657 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3f5f0b5{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-17T20:38:12,657 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-17T20:38:12,657 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@616d506d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-17T20:38:12,657 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@601b78f7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1a3196ea-489f-0fa1-8ed8-e2e462983867/hadoop.log.dir/,STOPPED}
2024-11-17T20:38:12,659 WARN  [BP-909746748-172.17.0.2-1731875851585 heartbeating to localhost/127.0.0.1:35449 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-17T20:38:12,659 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-17T20:38:12,659 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-17T20:38:12,659 WARN  [BP-909746748-172.17.0.2-1731875851585 heartbeating to localhost/127.0.0.1:35449 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-909746748-172.17.0.2-1731875851585 (Datanode Uuid a93d7341-9dda-4a25-be60-6e7cbeb82954) service to localhost/127.0.0.1:35449
2024-11-17T20:38:12,659 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1a3196ea-489f-0fa1-8ed8-e2e462983867/cluster_14a128d9-7999-521f-922f-c4fb1ce6abae/data/data1/current/BP-909746748-172.17.0.2-1731875851585 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T20:38:12,659 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1a3196ea-489f-0fa1-8ed8-e2e462983867/cluster_14a128d9-7999-521f-922f-c4fb1ce6abae/data/data2/current/BP-909746748-172.17.0.2-1731875851585 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T20:38:12,660 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-17T20:38:12,664 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5fe51576{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-17T20:38:12,665 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@284a8a01{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-17T20:38:12,665 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-17T20:38:12,665 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7193a060{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-17T20:38:12,665 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@444d0b71{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1a3196ea-489f-0fa1-8ed8-e2e462983867/hadoop.log.dir/,STOPPED}
2024-11-17T20:38:12,672 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-17T20:38:12,704 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-17T20:38:12,714 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRolling Thread=233 (was 209)
Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:35449
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:35449
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-41-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-38-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-40-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-39-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-14-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:35449 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-39-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-38-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:35449
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:35449
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-14-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-41-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-39-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:35449 from jenkins.hfs.6
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.6@localhost:35449
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-40-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-40-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-14-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-41-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:35449 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-38-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=521 (was 485) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=228 (was 221) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8233 (was 7430) - AvailableMemoryMB LEAK? -
2024-11-17T20:38:12,720 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnNothingWritten Thread=233, OpenFileDescriptor=521, MaxFileDescriptor=1048576, SystemLoadAverage=228, ProcessCount=11, AvailableMemoryMB=8233
2024-11-17T20:38:12,721 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-17T20:38:12,721 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1a3196ea-489f-0fa1-8ed8-e2e462983867/hadoop.log.dir so I do NOT create it in target/test-data/fdc09039-3224-6f0d-ffe0-8e19966578de
2024-11-17T20:38:12,721 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1a3196ea-489f-0fa1-8ed8-e2e462983867/hadoop.tmp.dir so I do NOT create it in target/test-data/fdc09039-3224-6f0d-ffe0-8e19966578de
2024-11-17T20:38:12,721 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fdc09039-3224-6f0d-ffe0-8e19966578de/cluster_0a835a85-a168-a15e-c665-c668c56cd27d, deleteOnExit=true
2024-11-17T20:38:12,721 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-17T20:38:12,721 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fdc09039-3224-6f0d-ffe0-8e19966578de/test.cache.data in system properties and HBase conf
2024-11-17T20:38:12,721 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fdc09039-3224-6f0d-ffe0-8e19966578de/hadoop.tmp.dir in system properties and HBase conf
2024-11-17T20:38:12,721 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fdc09039-3224-6f0d-ffe0-8e19966578de/hadoop.log.dir in system properties and HBase conf
2024-11-17T20:38:12,721 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fdc09039-3224-6f0d-ffe0-8e19966578de/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-17T20:38:12,721 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fdc09039-3224-6f0d-ffe0-8e19966578de/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-17T20:38:12,721 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-17T20:38:12,721 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-17T20:38:12,722 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fdc09039-3224-6f0d-ffe0-8e19966578de/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-17T20:38:12,722 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fdc09039-3224-6f0d-ffe0-8e19966578de/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-17T20:38:12,722 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fdc09039-3224-6f0d-ffe0-8e19966578de/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-17T20:38:12,722 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fdc09039-3224-6f0d-ffe0-8e19966578de/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-17T20:38:12,722 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fdc09039-3224-6f0d-ffe0-8e19966578de/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-17T20:38:12,722 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fdc09039-3224-6f0d-ffe0-8e19966578de/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-17T20:38:12,722 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fdc09039-3224-6f0d-ffe0-8e19966578de/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-17T20:38:12,722 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fdc09039-3224-6f0d-ffe0-8e19966578de/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-17T20:38:12,722 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fdc09039-3224-6f0d-ffe0-8e19966578de/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-17T20:38:12,722 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fdc09039-3224-6f0d-ffe0-8e19966578de/nfs.dump.dir in system properties and HBase conf
2024-11-17T20:38:12,722 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fdc09039-3224-6f0d-ffe0-8e19966578de/java.io.tmpdir in system properties and HBase conf
2024-11-17T20:38:12,722 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fdc09039-3224-6f0d-ffe0-8e19966578de/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-17T20:38:12,722 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fdc09039-3224-6f0d-ffe0-8e19966578de/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-17T20:38:12,722 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fdc09039-3224-6f0d-ffe0-8e19966578de/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-17T20:38:12,736 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-17T20:38:12,857 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:38:12,860 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:38:13,024 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:38:13,064 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-17T20:38:13,066 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-17T20:38:13,068 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-17T20:38:13,068 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-17T20:38:13,068 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-17T20:38:13,068 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-17T20:38:13,069 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3b55cb8{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fdc09039-3224-6f0d-ffe0-8e19966578de/hadoop.log.dir/,AVAILABLE}
2024-11-17T20:38:13,069 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4c4c959a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-17T20:38:13,159 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@12ac7c6{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fdc09039-3224-6f0d-ffe0-8e19966578de/java.io.tmpdir/jetty-localhost-36731-hadoop-hdfs-3_4_1-tests_jar-_-any-14616336597832250579/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-17T20:38:13,159 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2b13a29{HTTP/1.1, (http/1.1)}{localhost:36731}
2024-11-17T20:38:13,159 INFO  [Time-limited test {}] server.Server(415): Started @296445ms
2024-11-17T20:38:13,170 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-17T20:38:13,415 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-17T20:38:13,417 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-17T20:38:13,418 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-17T20:38:13,418 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-17T20:38:13,418 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-17T20:38:13,419 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2fb7753b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fdc09039-3224-6f0d-ffe0-8e19966578de/hadoop.log.dir/,AVAILABLE}
2024-11-17T20:38:13,419 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5f638c2a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-17T20:38:13,511 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@64440bf6{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fdc09039-3224-6f0d-ffe0-8e19966578de/java.io.tmpdir/jetty-localhost-39519-hadoop-hdfs-3_4_1-tests_jar-_-any-15718922717150476165/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T20:38:13,511 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2551ca75{HTTP/1.1, (http/1.1)}{localhost:39519}
2024-11-17T20:38:13,511 INFO  [Time-limited test {}] server.Server(415): Started @296797ms
2024-11-17T20:38:13,512 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-17T20:38:13,537 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-17T20:38:13,541 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-17T20:38:13,542 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-17T20:38:13,542 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-17T20:38:13,542 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-17T20:38:13,543 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6c2fc043{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fdc09039-3224-6f0d-ffe0-8e19966578de/hadoop.log.dir/,AVAILABLE}
2024-11-17T20:38:13,543 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@59a6b271{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-17T20:38:13,636 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3565f430{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fdc09039-3224-6f0d-ffe0-8e19966578de/java.io.tmpdir/jetty-localhost-35177-hadoop-hdfs-3_4_1-tests_jar-_-any-11713717414255561959/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T20:38:13,637 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@16ced564{HTTP/1.1, (http/1.1)}{localhost:35177}
2024-11-17T20:38:13,637 INFO  [Time-limited test {}] server.Server(415): Started @296923ms
2024-11-17T20:38:13,638 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-17T20:38:13,803 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-17T20:38:13,804 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-17T20:38:13,805 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-11-17T20:38:13,806 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling
2024-11-17T20:38:13,858 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:38:13,860 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:38:14,024 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:38:14,626 WARN  [Thread-2521 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fdc09039-3224-6f0d-ffe0-8e19966578de/cluster_0a835a85-a168-a15e-c665-c668c56cd27d/data/data1/current/BP-1884258858-172.17.0.2-1731875892739/current, will proceed with Du for space computation calculation, 
2024-11-17T20:38:14,627 WARN  [Thread-2522 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fdc09039-3224-6f0d-ffe0-8e19966578de/cluster_0a835a85-a168-a15e-c665-c668c56cd27d/data/data2/current/BP-1884258858-172.17.0.2-1731875892739/current, will proceed with Du for space computation calculation, 
2024-11-17T20:38:14,645 WARN  [Thread-2485 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-17T20:38:14,647 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x74c80bbaa9f1df9e with lease ID 0x281b1af563549dce: Processing first storage report for DS-c5e75833-da69-4ac8-b796-d19b7d37fb14 from datanode DatanodeRegistration(127.0.0.1:45183, datanodeUuid=b8e80017-b64e-4d84-9a5d-ddd073b2590a, infoPort=41597, infoSecurePort=0, ipcPort=44675, storageInfo=lv=-57;cid=testClusterID;nsid=246350220;c=1731875892739)
2024-11-17T20:38:14,647 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x74c80bbaa9f1df9e with lease ID 0x281b1af563549dce: from storage DS-c5e75833-da69-4ac8-b796-d19b7d37fb14 node DatanodeRegistration(127.0.0.1:45183, datanodeUuid=b8e80017-b64e-4d84-9a5d-ddd073b2590a, infoPort=41597, infoSecurePort=0, ipcPort=44675, storageInfo=lv=-57;cid=testClusterID;nsid=246350220;c=1731875892739), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-17T20:38:14,647 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x74c80bbaa9f1df9e with lease ID 0x281b1af563549dce: Processing first storage report for DS-38ace45f-3fa1-46b2-be07-0c10ce16e981 from datanode DatanodeRegistration(127.0.0.1:45183, datanodeUuid=b8e80017-b64e-4d84-9a5d-ddd073b2590a, infoPort=41597, infoSecurePort=0, ipcPort=44675, storageInfo=lv=-57;cid=testClusterID;nsid=246350220;c=1731875892739)
2024-11-17T20:38:14,647 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x74c80bbaa9f1df9e with lease ID 0x281b1af563549dce: from storage DS-38ace45f-3fa1-46b2-be07-0c10ce16e981 node DatanodeRegistration(127.0.0.1:45183, datanodeUuid=b8e80017-b64e-4d84-9a5d-ddd073b2590a, infoPort=41597, infoSecurePort=0, ipcPort=44675, storageInfo=lv=-57;cid=testClusterID;nsid=246350220;c=1731875892739), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-17T20:38:14,829 WARN  [Thread-2532 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fdc09039-3224-6f0d-ffe0-8e19966578de/cluster_0a835a85-a168-a15e-c665-c668c56cd27d/data/data3/current/BP-1884258858-172.17.0.2-1731875892739/current, will proceed with Du for space computation calculation, 
2024-11-17T20:38:14,829 WARN  [Thread-2533 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fdc09039-3224-6f0d-ffe0-8e19966578de/cluster_0a835a85-a168-a15e-c665-c668c56cd27d/data/data4/current/BP-1884258858-172.17.0.2-1731875892739/current, will proceed with Du for space computation calculation, 
2024-11-17T20:38:14,850 WARN  [Thread-2508 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-17T20:38:14,852 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa860d4136cb2ee7e with lease ID 0x281b1af563549dcf: Processing first storage report for DS-5b5b65f0-16d5-4d73-b012-4ed18890e5ee from datanode DatanodeRegistration(127.0.0.1:33139, datanodeUuid=328f7c29-1fee-4879-932b-76574a90d7bc, infoPort=34379, infoSecurePort=0, ipcPort=37969, storageInfo=lv=-57;cid=testClusterID;nsid=246350220;c=1731875892739)
2024-11-17T20:38:14,852 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa860d4136cb2ee7e with lease ID 0x281b1af563549dcf: from storage DS-5b5b65f0-16d5-4d73-b012-4ed18890e5ee node DatanodeRegistration(127.0.0.1:33139, datanodeUuid=328f7c29-1fee-4879-932b-76574a90d7bc, infoPort=34379, infoSecurePort=0, ipcPort=37969, storageInfo=lv=-57;cid=testClusterID;nsid=246350220;c=1731875892739), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-17T20:38:14,852 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa860d4136cb2ee7e with lease ID 0x281b1af563549dcf: Processing first storage report for DS-866dab0a-75d8-4e4d-a06b-9805af6d9a87 from datanode DatanodeRegistration(127.0.0.1:33139, datanodeUuid=328f7c29-1fee-4879-932b-76574a90d7bc, infoPort=34379, infoSecurePort=0, ipcPort=37969, storageInfo=lv=-57;cid=testClusterID;nsid=246350220;c=1731875892739)
2024-11-17T20:38:14,852 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa860d4136cb2ee7e with lease ID 0x281b1af563549dcf: from storage DS-866dab0a-75d8-4e4d-a06b-9805af6d9a87 node DatanodeRegistration(127.0.0.1:33139, datanodeUuid=328f7c29-1fee-4879-932b-76574a90d7bc, infoPort=34379, infoSecurePort=0, ipcPort=37969, storageInfo=lv=-57;cid=testClusterID;nsid=246350220;c=1731875892739), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-17T20:38:14,859 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:38:14,861 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:38:14,866 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fdc09039-3224-6f0d-ffe0-8e19966578de
2024-11-17T20:38:14,868 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fdc09039-3224-6f0d-ffe0-8e19966578de/cluster_0a835a85-a168-a15e-c665-c668c56cd27d/zookeeper_0, clientPort=52912, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fdc09039-3224-6f0d-ffe0-8e19966578de/cluster_0a835a85-a168-a15e-c665-c668c56cd27d/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fdc09039-3224-6f0d-ffe0-8e19966578de/cluster_0a835a85-a168-a15e-c665-c668c56cd27d/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-17T20:38:14,869 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=52912
2024-11-17T20:38:14,869 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T20:38:14,871 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T20:38:14,880 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33139 is added to blk_1073741825_1001 (size=7)
2024-11-17T20:38:14,880 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45183 is added to blk_1073741825_1001 (size=7)
2024-11-17T20:38:14,881 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:38271/user/jenkins/test-data/3d7d90a0-d2fc-ab12-ff81-43a91f4072b2 with version=8
2024-11-17T20:38:14,881 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:35585/user/jenkins/test-data/73fea5ca-777e-03b8-e712-189829aca80d/hbase-staging
2024-11-17T20:38:14,883 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/0b0b0ae8de03:0 server-side Connection retries=45
2024-11-17T20:38:14,883 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-17T20:38:14,883 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-17T20:38:14,883 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-17T20:38:14,883 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-17T20:38:14,883 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-17T20:38:14,883 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-17T20:38:14,883 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-17T20:38:14,884 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42011
2024-11-17T20:38:14,885 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:42011 connecting to ZooKeeper ensemble=127.0.0.1:52912
2024-11-17T20:38:15,025 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:38:15,135 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:420110x0, quorum=127.0.0.1:52912, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-17T20:38:15,136 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42011-0x1014a84f1ff0000 connected
2024-11-17T20:38:15,218 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T20:38:15,219 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T20:38:15,221 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42011-0x1014a84f1ff0000, quorum=127.0.0.1:52912, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-17T20:38:15,222 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:38271/user/jenkins/test-data/3d7d90a0-d2fc-ab12-ff81-43a91f4072b2, hbase.cluster.distributed=false
2024-11-17T20:38:15,224 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42011-0x1014a84f1ff0000, quorum=127.0.0.1:52912, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-17T20:38:15,224 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42011
2024-11-17T20:38:15,225 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42011
2024-11-17T20:38:15,225 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42011
2024-11-17T20:38:15,225 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42011
2024-11-17T20:38:15,226 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42011
2024-11-17T20:38:15,243 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/0b0b0ae8de03:0 server-side Connection retries=45
2024-11-17T20:38:15,244 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-17T20:38:15,244 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-17T20:38:15,244 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-17T20:38:15,244 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-17T20:38:15,244 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-17T20:38:15,244 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-17T20:38:15,244 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-17T20:38:15,244 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44343
2024-11-17T20:38:15,245 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44343 connecting to ZooKeeper ensemble=127.0.0.1:52912
2024-11-17T20:38:15,246 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T20:38:15,247 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T20:38:15,260 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:443430x0, quorum=127.0.0.1:52912, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-17T20:38:15,261 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44343-0x1014a84f1ff0001 connected
2024-11-17T20:38:15,261 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44343-0x1014a84f1ff0001, quorum=127.0.0.1:52912, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-17T20:38:15,261 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-17T20:38:15,262 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-17T20:38:15,262 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44343-0x1014a84f1ff0001, quorum=127.0.0.1:52912, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-17T20:38:15,263 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44343-0x1014a84f1ff0001, quorum=127.0.0.1:52912, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-17T20:38:15,264 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44343
2024-11-17T20:38:15,264 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44343
2024-11-17T20:38:15,265 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44343
2024-11-17T20:38:15,265 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44343
2024-11-17T20:38:15,265 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44343
2024-11-17T20:38:15,279 DEBUG [M:0;0b0b0ae8de03:42011 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;0b0b0ae8de03:42011
2024-11-17T20:38:15,279 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/0b0b0ae8de03,42011,1731875894882
2024-11-17T20:38:15,292 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42011-0x1014a84f1ff0000, quorum=127.0.0.1:52912, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-17T20:38:15,292 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44343-0x1014a84f1ff0001, quorum=127.0.0.1:52912, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-17T20:38:15,292 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42011-0x1014a84f1ff0000, quorum=127.0.0.1:52912, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/0b0b0ae8de03,42011,1731875894882
2024-11-17T20:38:15,302 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42011-0x1014a84f1ff0000, quorum=127.0.0.1:52912, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:38:15,302 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44343-0x1014a84f1ff0001, quorum=127.0.0.1:52912, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-17T20:38:15,302 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44343-0x1014a84f1ff0001, quorum=127.0.0.1:52912, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:38:15,303 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42011-0x1014a84f1ff0000, quorum=127.0.0.1:52912, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-17T20:38:15,304 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/0b0b0ae8de03,42011,1731875894882 from backup master directory
2024-11-17T20:38:15,313 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44343-0x1014a84f1ff0001, quorum=127.0.0.1:52912, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-17T20:38:15,313 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42011-0x1014a84f1ff0000, quorum=127.0.0.1:52912, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/0b0b0ae8de03,42011,1731875894882
2024-11-17T20:38:15,313 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42011-0x1014a84f1ff0000, quorum=127.0.0.1:52912, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-17T20:38:15,313 WARN  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-17T20:38:15,313 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=0b0b0ae8de03,42011,1731875894882
2024-11-17T20:38:15,320 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:38271/user/jenkins/test-data/3d7d90a0-d2fc-ab12-ff81-43a91f4072b2/hbase.id] with ID: 119e2ac1-89f8-4433-aaad-4851f5dfa120
2024-11-17T20:38:15,320 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:38271/user/jenkins/test-data/3d7d90a0-d2fc-ab12-ff81-43a91f4072b2/.tmp/hbase.id
2024-11-17T20:38:15,328 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33139 is added to blk_1073741826_1002 (size=42)
2024-11-17T20:38:15,328 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45183 is added to blk_1073741826_1002 (size=42)
2024-11-17T20:38:15,329 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:38271/user/jenkins/test-data/3d7d90a0-d2fc-ab12-ff81-43a91f4072b2/.tmp/hbase.id]:[hdfs://localhost:38271/user/jenkins/test-data/3d7d90a0-d2fc-ab12-ff81-43a91f4072b2/hbase.id]
2024-11-17T20:38:15,343 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T20:38:15,343 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-17T20:38:15,344 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms.
2024-11-17T20:38:15,426 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42011-0x1014a84f1ff0000, quorum=127.0.0.1:52912, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:38:15,426 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44343-0x1014a84f1ff0001, quorum=127.0.0.1:52912, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:38:15,437 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33139 is added to blk_1073741827_1003 (size=196)
2024-11-17T20:38:15,437 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45183 is added to blk_1073741827_1003 (size=196)
2024-11-17T20:38:15,438 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-17T20:38:15,439 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-17T20:38:15,440 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-17T20:38:15,446 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45183 is added to blk_1073741828_1004 (size=1189)
2024-11-17T20:38:15,446 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33139 is added to blk_1073741828_1004 (size=1189)
2024-11-17T20:38:15,447 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:38271/user/jenkins/test-data/3d7d90a0-d2fc-ab12-ff81-43a91f4072b2/MasterData/data/master/store
2024-11-17T20:38:15,452 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45183 is added to blk_1073741829_1005 (size=34)
2024-11-17T20:38:15,452 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33139 is added to blk_1073741829_1005 (size=34)
2024-11-17T20:38:15,452 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T20:38:15,453 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-17T20:38:15,453 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T20:38:15,453 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T20:38:15,453 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-17T20:38:15,453 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T20:38:15,453 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T20:38:15,453 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731875895452Disabling compacts and flushes for region at 1731875895452Disabling writes for close at 1731875895453 (+1 ms)Writing region close event to WAL at 1731875895453Closed at 1731875895453
2024-11-17T20:38:15,453 WARN  [master/0b0b0ae8de03:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:38271/user/jenkins/test-data/3d7d90a0-d2fc-ab12-ff81-43a91f4072b2/MasterData/data/master/store/.initializing
2024-11-17T20:38:15,453 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:38271/user/jenkins/test-data/3d7d90a0-d2fc-ab12-ff81-43a91f4072b2/MasterData/WALs/0b0b0ae8de03,42011,1731875894882
2024-11-17T20:38:15,456 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0b0b0ae8de03%2C42011%2C1731875894882, suffix=, logDir=hdfs://localhost:38271/user/jenkins/test-data/3d7d90a0-d2fc-ab12-ff81-43a91f4072b2/MasterData/WALs/0b0b0ae8de03,42011,1731875894882, archiveDir=hdfs://localhost:38271/user/jenkins/test-data/3d7d90a0-d2fc-ab12-ff81-43a91f4072b2/MasterData/oldWALs, maxLogs=10
2024-11-17T20:38:15,456 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0b0b0ae8de03%2C42011%2C1731875894882.1731875895456
2024-11-17T20:38:15,459 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3d7d90a0-d2fc-ab12-ff81-43a91f4072b2/MasterData/WALs/0b0b0ae8de03,42011,1731875894882/0b0b0ae8de03%2C42011%2C1731875894882.1731875895456
2024-11-17T20:38:15,460 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34379:34379),(127.0.0.1/127.0.0.1:41597:41597)]
2024-11-17T20:38:15,461 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-17T20:38:15,461 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T20:38:15,461 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T20:38:15,461 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T20:38:15,462 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-17T20:38:15,463 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-17T20:38:15,463 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:38:15,463 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T20:38:15,463 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-17T20:38:15,464 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-17T20:38:15,464 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:38:15,464 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-17T20:38:15,465 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-17T20:38:15,465 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-17T20:38:15,465 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:38:15,466 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-17T20:38:15,466 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-17T20:38:15,467 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-17T20:38:15,467 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:38:15,467 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-17T20:38:15,468 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T20:38:15,468 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38271/user/jenkins/test-data/3d7d90a0-d2fc-ab12-ff81-43a91f4072b2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-17T20:38:15,469 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38271/user/jenkins/test-data/3d7d90a0-d2fc-ab12-ff81-43a91f4072b2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-17T20:38:15,470 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T20:38:15,470 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T20:38:15,470 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-17T20:38:15,471 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T20:38:15,473 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38271/user/jenkins/test-data/3d7d90a0-d2fc-ab12-ff81-43a91f4072b2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-17T20:38:15,473 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=741877, jitterRate=-0.056655243039131165}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-17T20:38:15,474 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1731875895461Initializing all the Stores at 1731875895462 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731875895462Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731875895462Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731875895462Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731875895462Cleaning up temporary data from old regions at 1731875895470 (+8 ms)Region opened successfully at 1731875895474 (+4 ms)
2024-11-17T20:38:15,474 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-17T20:38:15,477 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7dfbad40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0b0b0ae8de03/172.17.0.2:0
2024-11-17T20:38:15,478 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-17T20:38:15,478 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-17T20:38:15,478 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-17T20:38:15,478 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-17T20:38:15,479 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-17T20:38:15,479 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-17T20:38:15,479 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-17T20:38:15,481 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-17T20:38:15,482 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42011-0x1014a84f1ff0000, quorum=127.0.0.1:52912, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-17T20:38:15,513 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-17T20:38:15,513 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-17T20:38:15,514 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42011-0x1014a84f1ff0000, quorum=127.0.0.1:52912, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-17T20:38:15,523 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-17T20:38:15,524 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-17T20:38:15,525 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42011-0x1014a84f1ff0000, quorum=127.0.0.1:52912, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-17T20:38:15,534 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-17T20:38:15,535 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42011-0x1014a84f1ff0000, quorum=127.0.0.1:52912, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-17T20:38:15,544 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-17T20:38:15,547 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42011-0x1014a84f1ff0000, quorum=127.0.0.1:52912, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-17T20:38:15,555 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-17T20:38:15,565 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42011-0x1014a84f1ff0000, quorum=127.0.0.1:52912, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-17T20:38:15,565 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44343-0x1014a84f1ff0001, quorum=127.0.0.1:52912, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-17T20:38:15,565 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42011-0x1014a84f1ff0000, quorum=127.0.0.1:52912, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:38:15,565 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44343-0x1014a84f1ff0001, quorum=127.0.0.1:52912, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:38:15,566 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=0b0b0ae8de03,42011,1731875894882, sessionid=0x1014a84f1ff0000, setting cluster-up flag (Was=false)
2024-11-17T20:38:15,586 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42011-0x1014a84f1ff0000, quorum=127.0.0.1:52912, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:38:15,587 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44343-0x1014a84f1ff0001, quorum=127.0.0.1:52912, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:38:15,618 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-17T20:38:15,619 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0b0b0ae8de03,42011,1731875894882
2024-11-17T20:38:15,639 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42011-0x1014a84f1ff0000, quorum=127.0.0.1:52912, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:38:15,639 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44343-0x1014a84f1ff0001, quorum=127.0.0.1:52912, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:38:15,671 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-17T20:38:15,674 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0b0b0ae8de03,42011,1731875894882
2024-11-17T20:38:15,676 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:38271/user/jenkins/test-data/3d7d90a0-d2fc-ab12-ff81-43a91f4072b2/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-17T20:38:15,678 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-17T20:38:15,679 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-17T20:38:15,679 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-17T20:38:15,679 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 0b0b0ae8de03,42011,1731875894882
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-17T20:38:15,681 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/0b0b0ae8de03:0, corePoolSize=5, maxPoolSize=5
2024-11-17T20:38:15,681 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/0b0b0ae8de03:0, corePoolSize=5, maxPoolSize=5
2024-11-17T20:38:15,681 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/0b0b0ae8de03:0, corePoolSize=5, maxPoolSize=5
2024-11-17T20:38:15,681 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/0b0b0ae8de03:0, corePoolSize=5, maxPoolSize=5
2024-11-17T20:38:15,682 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/0b0b0ae8de03:0, corePoolSize=10, maxPoolSize=10
2024-11-17T20:38:15,682 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:38:15,682 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/0b0b0ae8de03:0, corePoolSize=2, maxPoolSize=2
2024-11-17T20:38:15,682 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:38:15,683 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731875925683
2024-11-17T20:38:15,683 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-17T20:38:15,683 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-17T20:38:15,683 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-17T20:38:15,683 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-17T20:38:15,683 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-17T20:38:15,683 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-17T20:38:15,684 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-17T20:38:15,684 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-17T20:38:15,684 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-17T20:38:15,684 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-17T20:38:15,684 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-17T20:38:15,684 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-17T20:38:15,685 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-17T20:38:15,685 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-17T20:38:15,685 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/0b0b0ae8de03:0:becomeActiveMaster-HFileCleaner.large.0-1731875895685,5,FailOnTimeoutGroup]
2024-11-17T20:38:15,685 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/0b0b0ae8de03:0:becomeActiveMaster-HFileCleaner.small.0-1731875895685,5,FailOnTimeoutGroup]
2024-11-17T20:38:15,685 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-17T20:38:15,685 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-17T20:38:15,685 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-17T20:38:15,686 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:38:15,686 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-17T20:38:15,686 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-17T20:38:15,693 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33139 is added to blk_1073741831_1007 (size=1321)
2024-11-17T20:38:15,693 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45183 is added to blk_1073741831_1007 (size=1321)
2024-11-17T20:38:15,694 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:38271/user/jenkins/test-data/3d7d90a0-d2fc-ab12-ff81-43a91f4072b2/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-17T20:38:15,694 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:38271/user/jenkins/test-data/3d7d90a0-d2fc-ab12-ff81-43a91f4072b2
2024-11-17T20:38:15,699 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33139 is added to blk_1073741832_1008 (size=32)
2024-11-17T20:38:15,700 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45183 is added to blk_1073741832_1008 (size=32)
2024-11-17T20:38:15,700 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T20:38:15,701 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-17T20:38:15,702 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-17T20:38:15,702 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:38:15,702 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T20:38:15,702 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-17T20:38:15,703 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-17T20:38:15,703 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:38:15,703 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T20:38:15,704 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-17T20:38:15,704 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-17T20:38:15,704 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:38:15,705 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T20:38:15,705 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-17T20:38:15,705 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-17T20:38:15,706 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:38:15,706 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T20:38:15,706 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-17T20:38:15,707 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38271/user/jenkins/test-data/3d7d90a0-d2fc-ab12-ff81-43a91f4072b2/data/hbase/meta/1588230740
2024-11-17T20:38:15,707 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38271/user/jenkins/test-data/3d7d90a0-d2fc-ab12-ff81-43a91f4072b2/data/hbase/meta/1588230740
2024-11-17T20:38:15,708 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-17T20:38:15,708 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-17T20:38:15,708 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-17T20:38:15,709 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-17T20:38:15,711 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38271/user/jenkins/test-data/3d7d90a0-d2fc-ab12-ff81-43a91f4072b2/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-17T20:38:15,711 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=776376, jitterRate=-0.012787774205207825}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-17T20:38:15,712 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1731875895700Initializing all the Stores at 1731875895701 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731875895701Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731875895701Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731875895701Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731875895701Cleaning up temporary data from old regions at 1731875895708 (+7 ms)Region opened successfully at 1731875895712 (+4 ms)
2024-11-17T20:38:15,712 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-17T20:38:15,712 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-17T20:38:15,712 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-17T20:38:15,712 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-17T20:38:15,712 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-17T20:38:15,712 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-17T20:38:15,712 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731875895712Disabling compacts and flushes for region at 1731875895712Disabling writes for close at 1731875895712Writing region close event to WAL at 1731875895712Closed at 1731875895712
2024-11-17T20:38:15,713 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-17T20:38:15,713 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-17T20:38:15,713 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-17T20:38:15,714 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-17T20:38:15,715 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-17T20:38:15,767 INFO  [RS:0;0b0b0ae8de03:44343 {}] regionserver.HRegionServer(746): ClusterId : 119e2ac1-89f8-4433-aaad-4851f5dfa120
2024-11-17T20:38:15,767 DEBUG [RS:0;0b0b0ae8de03:44343 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-17T20:38:15,777 DEBUG [RS:0;0b0b0ae8de03:44343 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-17T20:38:15,778 DEBUG [RS:0;0b0b0ae8de03:44343 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-17T20:38:15,788 DEBUG [RS:0;0b0b0ae8de03:44343 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-17T20:38:15,788 DEBUG [RS:0;0b0b0ae8de03:44343 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3550a34a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0b0b0ae8de03/172.17.0.2:0
2024-11-17T20:38:15,804 DEBUG [RS:0;0b0b0ae8de03:44343 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;0b0b0ae8de03:44343
2024-11-17T20:38:15,804 INFO  [RS:0;0b0b0ae8de03:44343 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-17T20:38:15,804 INFO  [RS:0;0b0b0ae8de03:44343 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-17T20:38:15,804 DEBUG [RS:0;0b0b0ae8de03:44343 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-17T20:38:15,805 INFO  [RS:0;0b0b0ae8de03:44343 {}] regionserver.HRegionServer(2659): reportForDuty to master=0b0b0ae8de03,42011,1731875894882 with port=44343, startcode=1731875895243
2024-11-17T20:38:15,805 DEBUG [RS:0;0b0b0ae8de03:44343 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-17T20:38:15,807 INFO  [HMaster-EventLoopGroup-16-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50791, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.7 (auth:SIMPLE), service=RegionServerStatusService
2024-11-17T20:38:15,807 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42011 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 0b0b0ae8de03,44343,1731875895243
2024-11-17T20:38:15,807 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42011 {}] master.ServerManager(517): Registering regionserver=0b0b0ae8de03,44343,1731875895243
2024-11-17T20:38:15,809 DEBUG [RS:0;0b0b0ae8de03:44343 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38271/user/jenkins/test-data/3d7d90a0-d2fc-ab12-ff81-43a91f4072b2
2024-11-17T20:38:15,809 DEBUG [RS:0;0b0b0ae8de03:44343 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38271
2024-11-17T20:38:15,809 DEBUG [RS:0;0b0b0ae8de03:44343 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-17T20:38:15,818 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42011-0x1014a84f1ff0000, quorum=127.0.0.1:52912, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-17T20:38:15,819 DEBUG [RS:0;0b0b0ae8de03:44343 {}] zookeeper.ZKUtil(111): regionserver:44343-0x1014a84f1ff0001, quorum=127.0.0.1:52912, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/0b0b0ae8de03,44343,1731875895243
2024-11-17T20:38:15,819 WARN  [RS:0;0b0b0ae8de03:44343 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-17T20:38:15,819 INFO  [RS:0;0b0b0ae8de03:44343 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-17T20:38:15,819 DEBUG [RS:0;0b0b0ae8de03:44343 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38271/user/jenkins/test-data/3d7d90a0-d2fc-ab12-ff81-43a91f4072b2/WALs/0b0b0ae8de03,44343,1731875895243
2024-11-17T20:38:15,819 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [0b0b0ae8de03,44343,1731875895243]
2024-11-17T20:38:15,822 INFO  [RS:0;0b0b0ae8de03:44343 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-17T20:38:15,824 INFO  [RS:0;0b0b0ae8de03:44343 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-17T20:38:15,824 INFO  [RS:0;0b0b0ae8de03:44343 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-17T20:38:15,824 INFO  [RS:0;0b0b0ae8de03:44343 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-17T20:38:15,825 INFO  [RS:0;0b0b0ae8de03:44343 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-17T20:38:15,825 INFO  [RS:0;0b0b0ae8de03:44343 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-17T20:38:15,825 INFO  [RS:0;0b0b0ae8de03:44343 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-17T20:38:15,826 DEBUG [RS:0;0b0b0ae8de03:44343 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:38:15,826 DEBUG [RS:0;0b0b0ae8de03:44343 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:38:15,826 DEBUG [RS:0;0b0b0ae8de03:44343 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:38:15,826 DEBUG [RS:0;0b0b0ae8de03:44343 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:38:15,826 DEBUG [RS:0;0b0b0ae8de03:44343 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:38:15,826 DEBUG [RS:0;0b0b0ae8de03:44343 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/0b0b0ae8de03:0, corePoolSize=2, maxPoolSize=2
2024-11-17T20:38:15,826 DEBUG [RS:0;0b0b0ae8de03:44343 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:38:15,826 DEBUG [RS:0;0b0b0ae8de03:44343 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:38:15,826 DEBUG [RS:0;0b0b0ae8de03:44343 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:38:15,826 DEBUG [RS:0;0b0b0ae8de03:44343 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:38:15,826 DEBUG [RS:0;0b0b0ae8de03:44343 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:38:15,826 DEBUG [RS:0;0b0b0ae8de03:44343 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1
2024-11-17T20:38:15,826 DEBUG [RS:0;0b0b0ae8de03:44343 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/0b0b0ae8de03:0, corePoolSize=3, maxPoolSize=3
2024-11-17T20:38:15,826 DEBUG [RS:0;0b0b0ae8de03:44343 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0, corePoolSize=3, maxPoolSize=3
2024-11-17T20:38:15,827 INFO  [RS:0;0b0b0ae8de03:44343 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-17T20:38:15,827 INFO  [RS:0;0b0b0ae8de03:44343 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-17T20:38:15,827 INFO  [RS:0;0b0b0ae8de03:44343 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-17T20:38:15,827 INFO  [RS:0;0b0b0ae8de03:44343 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-17T20:38:15,827 INFO  [RS:0;0b0b0ae8de03:44343 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-17T20:38:15,827 INFO  [RS:0;0b0b0ae8de03:44343 {}] hbase.ChoreService(168): Chore ScheduledChore name=0b0b0ae8de03,44343,1731875895243-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-17T20:38:15,847 INFO  [RS:0;0b0b0ae8de03:44343 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-17T20:38:15,848 INFO  [RS:0;0b0b0ae8de03:44343 {}] hbase.ChoreService(168): Chore ScheduledChore name=0b0b0ae8de03,44343,1731875895243-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-17T20:38:15,848 INFO  [RS:0;0b0b0ae8de03:44343 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T20:38:15,848 INFO  [RS:0;0b0b0ae8de03:44343 {}] regionserver.Replication(171): 0b0b0ae8de03,44343,1731875895243 started
2024-11-17T20:38:15,859 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:38:15,862 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:38:15,865 WARN  [0b0b0ae8de03:42011 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions.
2024-11-17T20:38:15,866 INFO  [RS:0;0b0b0ae8de03:44343 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T20:38:15,866 INFO  [RS:0;0b0b0ae8de03:44343 {}] regionserver.HRegionServer(1482): Serving as 0b0b0ae8de03,44343,1731875895243, RpcServer on 0b0b0ae8de03/172.17.0.2:44343, sessionid=0x1014a84f1ff0001
2024-11-17T20:38:15,866 DEBUG [RS:0;0b0b0ae8de03:44343 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-17T20:38:15,866 DEBUG [RS:0;0b0b0ae8de03:44343 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 0b0b0ae8de03,44343,1731875895243
2024-11-17T20:38:15,866 DEBUG [RS:0;0b0b0ae8de03:44343 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0b0b0ae8de03,44343,1731875895243'
2024-11-17T20:38:15,866 DEBUG [RS:0;0b0b0ae8de03:44343 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-17T20:38:15,867 DEBUG [RS:0;0b0b0ae8de03:44343 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-17T20:38:15,867 DEBUG [RS:0;0b0b0ae8de03:44343 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-17T20:38:15,867 DEBUG [RS:0;0b0b0ae8de03:44343 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-17T20:38:15,867 DEBUG [RS:0;0b0b0ae8de03:44343 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 0b0b0ae8de03,44343,1731875895243
2024-11-17T20:38:15,868 DEBUG [RS:0;0b0b0ae8de03:44343 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0b0b0ae8de03,44343,1731875895243'
2024-11-17T20:38:15,868 DEBUG [RS:0;0b0b0ae8de03:44343 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-17T20:38:15,868 DEBUG [RS:0;0b0b0ae8de03:44343 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-17T20:38:15,868 DEBUG [RS:0;0b0b0ae8de03:44343 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-17T20:38:15,868 INFO  [RS:0;0b0b0ae8de03:44343 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-17T20:38:15,868 INFO  [RS:0;0b0b0ae8de03:44343 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-17T20:38:15,971 INFO  [RS:0;0b0b0ae8de03:44343 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0b0b0ae8de03%2C44343%2C1731875895243, suffix=, logDir=hdfs://localhost:38271/user/jenkins/test-data/3d7d90a0-d2fc-ab12-ff81-43a91f4072b2/WALs/0b0b0ae8de03,44343,1731875895243, archiveDir=hdfs://localhost:38271/user/jenkins/test-data/3d7d90a0-d2fc-ab12-ff81-43a91f4072b2/oldWALs, maxLogs=32
2024-11-17T20:38:15,972 INFO  [RS:0;0b0b0ae8de03:44343 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0b0b0ae8de03%2C44343%2C1731875895243.1731875895972
2024-11-17T20:38:15,979 INFO  [RS:0;0b0b0ae8de03:44343 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3d7d90a0-d2fc-ab12-ff81-43a91f4072b2/WALs/0b0b0ae8de03,44343,1731875895243/0b0b0ae8de03%2C44343%2C1731875895243.1731875895972
2024-11-17T20:38:15,980 DEBUG [RS:0;0b0b0ae8de03:44343 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41597:41597),(127.0.0.1/127.0.0.1:34379:34379)]
2024-11-17T20:38:16,026 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:38:16,115 DEBUG [0b0b0ae8de03:42011 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-17T20:38:16,116 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=0b0b0ae8de03,44343,1731875895243
2024-11-17T20:38:16,118 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0b0b0ae8de03,44343,1731875895243, state=OPENING
2024-11-17T20:38:16,130 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-17T20:38:16,219 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44343-0x1014a84f1ff0001, quorum=127.0.0.1:52912, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:38:16,219 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42011-0x1014a84f1ff0000, quorum=127.0.0.1:52912, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:38:16,221 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-17T20:38:16,221 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-17T20:38:16,221 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-17T20:38:16,221 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=0b0b0ae8de03,44343,1731875895243}]
2024-11-17T20:38:16,377 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-17T20:38:16,382 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-17-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41167, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-17T20:38:16,386 INFO  [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-17T20:38:16,387 INFO  [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-17T20:38:16,389 INFO  [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0b0b0ae8de03%2C44343%2C1731875895243.meta, suffix=.meta, logDir=hdfs://localhost:38271/user/jenkins/test-data/3d7d90a0-d2fc-ab12-ff81-43a91f4072b2/WALs/0b0b0ae8de03,44343,1731875895243, archiveDir=hdfs://localhost:38271/user/jenkins/test-data/3d7d90a0-d2fc-ab12-ff81-43a91f4072b2/oldWALs, maxLogs=32
2024-11-17T20:38:16,390 INFO  [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 0b0b0ae8de03%2C44343%2C1731875895243.meta.1731875896390.meta
2024-11-17T20:38:16,395 INFO  [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3d7d90a0-d2fc-ab12-ff81-43a91f4072b2/WALs/0b0b0ae8de03,44343,1731875895243/0b0b0ae8de03%2C44343%2C1731875895243.meta.1731875896390.meta
2024-11-17T20:38:16,400 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41597:41597),(127.0.0.1/127.0.0.1:34379:34379)]
2024-11-17T20:38:16,404 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-17T20:38:16,404 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-17T20:38:16,404 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-17T20:38:16,405 INFO  [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-17T20:38:16,405 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-17T20:38:16,405 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T20:38:16,405 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-17T20:38:16,405 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-17T20:38:16,406 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-17T20:38:16,407 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-17T20:38:16,407 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:38:16,407 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T20:38:16,407 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-17T20:38:16,408 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-17T20:38:16,408 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:38:16,408 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T20:38:16,409 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-17T20:38:16,409 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-17T20:38:16,409 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:38:16,410 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T20:38:16,410 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-17T20:38:16,410 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-17T20:38:16,410 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T20:38:16,411 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T20:38:16,411 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-17T20:38:16,411 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38271/user/jenkins/test-data/3d7d90a0-d2fc-ab12-ff81-43a91f4072b2/data/hbase/meta/1588230740
2024-11-17T20:38:16,412 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38271/user/jenkins/test-data/3d7d90a0-d2fc-ab12-ff81-43a91f4072b2/data/hbase/meta/1588230740
2024-11-17T20:38:16,413 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-17T20:38:16,413 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-17T20:38:16,414 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-17T20:38:16,415 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-17T20:38:16,416 INFO  [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=849049, jitterRate=0.07962191104888916}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-17T20:38:16,416 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-17T20:38:16,416 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1731875896405Writing region info on filesystem at 1731875896405Initializing all the Stores at 1731875896406 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731875896406Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731875896406Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731875896406Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731875896406Cleaning up temporary data from old regions at 1731875896413 (+7 ms)Running coprocessor post-open hooks at 1731875896416 (+3 ms)Region opened successfully at 1731875896416
2024-11-17T20:38:16,417 INFO  [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731875896377
2024-11-17T20:38:16,419 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-17T20:38:16,419 INFO  [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-17T20:38:16,420 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=0b0b0ae8de03,44343,1731875895243
2024-11-17T20:38:16,421 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0b0b0ae8de03,44343,1731875895243, state=OPEN
2024-11-17T20:38:16,456 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42011-0x1014a84f1ff0000, quorum=127.0.0.1:52912, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-17T20:38:16,456 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44343-0x1014a84f1ff0001, quorum=127.0.0.1:52912, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-17T20:38:16,457 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=0b0b0ae8de03,44343,1731875895243
2024-11-17T20:38:16,457 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-17T20:38:16,457 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-17T20:38:16,460 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-17T20:38:16,461 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=0b0b0ae8de03,44343,1731875895243 in 236 msec
2024-11-17T20:38:16,465 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-17T20:38:16,465 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 748 msec
2024-11-17T20:38:16,466 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-17T20:38:16,466 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-17T20:38:16,468 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-17T20:38:16,468 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,44343,1731875895243, seqNum=-1]
2024-11-17T20:38:16,468 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-17T20:38:16,470 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-17-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40003, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-17T20:38:16,476 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 797 msec
2024-11-17T20:38:16,476 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731875896476, completionTime=-1
2024-11-17T20:38:16,477 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-17T20:38:16,477 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster...
2024-11-17T20:38:16,480 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1
2024-11-17T20:38:16,480 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731875956480
2024-11-17T20:38:16,480 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731876016480
2024-11-17T20:38:16,480 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 3 msec
2024-11-17T20:38:16,480 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0b0b0ae8de03,42011,1731875894882-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-17T20:38:16,480 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0b0b0ae8de03,42011,1731875894882-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T20:38:16,480 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0b0b0ae8de03,42011,1731875894882-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T20:38:16,480 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-0b0b0ae8de03:42011, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T20:38:16,480 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-17T20:38:16,481 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-17T20:38:16,483 DEBUG [master/0b0b0ae8de03:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-17T20:38:16,487 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.174sec
2024-11-17T20:38:16,487 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-17T20:38:16,487 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-17T20:38:16,487 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-17T20:38:16,487 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-17T20:38:16,487 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-17T20:38:16,487 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0b0b0ae8de03,42011,1731875894882-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-17T20:38:16,487 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0b0b0ae8de03,42011,1731875894882-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-17T20:38:16,490 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-17T20:38:16,490 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-17T20:38:16,491 INFO  [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0b0b0ae8de03,42011,1731875894882-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T20:38:16,568 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2187d3bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-17T20:38:16,568 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b0b0ae8de03,42011,-1 for getting cluster id
2024-11-17T20:38:16,569 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-17T20:38:16,570 DEBUG [HMaster-EventLoopGroup-16-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '119e2ac1-89f8-4433-aaad-4851f5dfa120'
2024-11-17T20:38:16,571 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-17T20:38:16,571 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "119e2ac1-89f8-4433-aaad-4851f5dfa120"

2024-11-17T20:38:16,571 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1be746b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-17T20:38:16,571 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b0b0ae8de03,42011,-1]
2024-11-17T20:38:16,572 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-17T20:38:16,572 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T20:38:16,573 INFO  [HMaster-EventLoopGroup-16-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57696, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-17T20:38:16,575 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15b1f8cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-17T20:38:16,575 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-17T20:38:16,577 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,44343,1731875895243, seqNum=-1]
2024-11-17T20:38:16,577 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-17T20:38:16,579 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-17-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59142, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-17T20:38:16,581 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=0b0b0ae8de03,42011,1731875894882
2024-11-17T20:38:16,581 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T20:38:16,585 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-17T20:38:16,585 INFO  [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-17T20:38:16,587 INFO  [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=test.com%2C8080%2C1, suffix=, logDir=hdfs://localhost:38271/user/jenkins/test-data/3d7d90a0-d2fc-ab12-ff81-43a91f4072b2/WALs/test.com,8080,1, archiveDir=hdfs://localhost:38271/user/jenkins/test-data/3d7d90a0-d2fc-ab12-ff81-43a91f4072b2/oldWALs, maxLogs=32
2024-11-17T20:38:16,588 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor test.com%2C8080%2C1.1731875896588
2024-11-17T20:38:16,593 INFO  [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3d7d90a0-d2fc-ab12-ff81-43a91f4072b2/WALs/test.com,8080,1/test.com%2C8080%2C1.1731875896588
2024-11-17T20:38:16,595 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41597:41597),(127.0.0.1/127.0.0.1:34379:34379)]
2024-11-17T20:38:16,596 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor test.com%2C8080%2C1.1731875896595
2024-11-17T20:38:16,602 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:38:16,602 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:38:16,602 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:38:16,602 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:38:16,602 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:38:16,602 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/3d7d90a0-d2fc-ab12-ff81-43a91f4072b2/WALs/test.com,8080,1/test.com%2C8080%2C1.1731875896588 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/3d7d90a0-d2fc-ab12-ff81-43a91f4072b2/WALs/test.com,8080,1/test.com%2C8080%2C1.1731875896595
2024-11-17T20:38:16,603 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34379:34379),(127.0.0.1/127.0.0.1:41597:41597)]
2024-11-17T20:38:16,603 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:38271/user/jenkins/test-data/3d7d90a0-d2fc-ab12-ff81-43a91f4072b2/WALs/test.com,8080,1/test.com%2C8080%2C1.1731875896588 is not closed yet, will try archiving it next time
2024-11-17T20:38:16,604 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:38:16,604 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:38:16,604 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33139 is added to blk_1073741835_1011 (size=93)
2024-11-17T20:38:16,604 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:38:16,604 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:38:16,605 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:38:16,605 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45183 is added to blk_1073741835_1011 (size=93)
2024-11-17T20:38:16,606 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:38271/user/jenkins/test-data/3d7d90a0-d2fc-ab12-ff81-43a91f4072b2/WALs/test.com,8080,1/test.com%2C8080%2C1.1731875896588 to hdfs://localhost:38271/user/jenkins/test-data/3d7d90a0-d2fc-ab12-ff81-43a91f4072b2/oldWALs/test.com%2C8080%2C1.1731875896588
2024-11-17T20:38:16,607 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45183 is added to blk_1073741836_1012 (size=93)
2024-11-17T20:38:16,607 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33139 is added to blk_1073741836_1012 (size=93)
2024-11-17T20:38:16,609 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/3d7d90a0-d2fc-ab12-ff81-43a91f4072b2/oldWALs
2024-11-17T20:38:16,609 INFO  [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog test.com%2C8080%2C1:(num 1731875896595)
2024-11-17T20:38:16,609 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-17T20:38:16,609 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-17T20:38:16,610 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-17T20:38:16,610 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T20:38:16,610 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T20:38:16,610 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-17T20:38:16,610 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-17T20:38:16,610 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=568381969, stopped=false
2024-11-17T20:38:16,610 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=0b0b0ae8de03,42011,1731875894882
2024-11-17T20:38:16,629 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42011-0x1014a84f1ff0000, quorum=127.0.0.1:52912, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-17T20:38:16,629 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44343-0x1014a84f1ff0001, quorum=127.0.0.1:52912, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-17T20:38:16,629 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42011-0x1014a84f1ff0000, quorum=127.0.0.1:52912, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:38:16,629 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44343-0x1014a84f1ff0001, quorum=127.0.0.1:52912, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:38:16,629 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-17T20:38:16,629 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-17T20:38:16,629 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-17T20:38:16,630 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T20:38:16,630 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42011-0x1014a84f1ff0000, quorum=127.0.0.1:52912, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-17T20:38:16,630 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '0b0b0ae8de03,44343,1731875895243' *****
2024-11-17T20:38:16,630 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44343-0x1014a84f1ff0001, quorum=127.0.0.1:52912, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-17T20:38:16,630 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-17T20:38:16,630 INFO  [RS:0;0b0b0ae8de03:44343 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-17T20:38:16,631 INFO  [RS:0;0b0b0ae8de03:44343 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-17T20:38:16,631 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-17T20:38:16,631 INFO  [RS:0;0b0b0ae8de03:44343 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-17T20:38:16,631 INFO  [RS:0;0b0b0ae8de03:44343 {}] regionserver.HRegionServer(959): stopping server 0b0b0ae8de03,44343,1731875895243
2024-11-17T20:38:16,631 INFO  [RS:0;0b0b0ae8de03:44343 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-17T20:38:16,631 INFO  [RS:0;0b0b0ae8de03:44343 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;0b0b0ae8de03:44343.
2024-11-17T20:38:16,631 DEBUG [RS:0;0b0b0ae8de03:44343 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-17T20:38:16,631 DEBUG [RS:0;0b0b0ae8de03:44343 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T20:38:16,632 INFO  [RS:0;0b0b0ae8de03:44343 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-17T20:38:16,632 INFO  [RS:0;0b0b0ae8de03:44343 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-17T20:38:16,632 INFO  [RS:0;0b0b0ae8de03:44343 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-17T20:38:16,632 INFO  [RS:0;0b0b0ae8de03:44343 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-17T20:38:16,632 INFO  [RS:0;0b0b0ae8de03:44343 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close
2024-11-17T20:38:16,632 DEBUG [RS:0;0b0b0ae8de03:44343 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740}
2024-11-17T20:38:16,632 DEBUG [RS:0;0b0b0ae8de03:44343 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-17T20:38:16,633 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-17T20:38:16,633 INFO  [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-17T20:38:16,633 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-17T20:38:16,633 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-17T20:38:16,633 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-17T20:38:16,633 INFO  [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=74 B heapSize=1.22 KB
2024-11-17T20:38:16,650 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38271/user/jenkins/test-data/3d7d90a0-d2fc-ab12-ff81-43a91f4072b2/data/hbase/meta/1588230740/.tmp/ns/5ce40735caf64d69a23a50cc8aa7c764 is 43, key is default/ns:d/1731875896470/Put/seqid=0
2024-11-17T20:38:16,654 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33139 is added to blk_1073741837_1013 (size=5153)
2024-11-17T20:38:16,654 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45183 is added to blk_1073741837_1013 (size=5153)
2024-11-17T20:38:16,655 INFO  [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:38271/user/jenkins/test-data/3d7d90a0-d2fc-ab12-ff81-43a91f4072b2/data/hbase/meta/1588230740/.tmp/ns/5ce40735caf64d69a23a50cc8aa7c764
2024-11-17T20:38:16,660 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38271/user/jenkins/test-data/3d7d90a0-d2fc-ab12-ff81-43a91f4072b2/data/hbase/meta/1588230740/.tmp/ns/5ce40735caf64d69a23a50cc8aa7c764 as hdfs://localhost:38271/user/jenkins/test-data/3d7d90a0-d2fc-ab12-ff81-43a91f4072b2/data/hbase/meta/1588230740/ns/5ce40735caf64d69a23a50cc8aa7c764
2024-11-17T20:38:16,665 INFO  [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38271/user/jenkins/test-data/3d7d90a0-d2fc-ab12-ff81-43a91f4072b2/data/hbase/meta/1588230740/ns/5ce40735caf64d69a23a50cc8aa7c764, entries=2, sequenceid=6, filesize=5.0 K
2024-11-17T20:38:16,666 INFO  [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 33ms, sequenceid=6, compaction requested=false
2024-11-17T20:38:16,666 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-17T20:38:16,670 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38271/user/jenkins/test-data/3d7d90a0-d2fc-ab12-ff81-43a91f4072b2/data/hbase/meta/1588230740/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1
2024-11-17T20:38:16,670 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-17T20:38:16,671 INFO  [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-17T20:38:16,671 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731875896632Running coprocessor pre-close hooks at 1731875896632Disabling compacts and flushes for region at 1731875896632Disabling writes for close at 1731875896633 (+1 ms)Obtaining lock to block concurrent updates at 1731875896633Preparing flush snapshotting stores in 1588230740 at 1731875896633Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=74, getHeapSize=1184, getOffHeapSize=0, getCellsCount=2 at 1731875896633Flushing stores of hbase:meta,,1.1588230740 at 1731875896634 (+1 ms)Flushing 1588230740/ns: creating writer at 1731875896635 (+1 ms)Flushing 1588230740/ns: appending metadata at 1731875896650 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1731875896650Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4cc12bbe: reopening flushed file at 1731875896660 (+10 ms)Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 33ms, sequenceid=6, compaction requested=false at 1731875896666 (+6 ms)Writing region close event to WAL at 1731875896667 (+1 ms)Running coprocessor post-close hooks at 1731875896670 (+3 ms)Closed at 1731875896671 (+1 ms)
2024-11-17T20:38:16,671 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-17T20:38:16,833 INFO  [RS:0;0b0b0ae8de03:44343 {}] regionserver.HRegionServer(976): stopping server 0b0b0ae8de03,44343,1731875895243; all regions closed.
2024-11-17T20:38:16,834 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:38:16,834 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:38:16,834 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:38:16,835 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:38:16,835 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:38:16,839 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33139 is added to blk_1073741834_1010 (size=1152)
2024-11-17T20:38:16,840 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45183 is added to blk_1073741834_1010 (size=1152)
2024-11-17T20:38:16,845 DEBUG [RS:0;0b0b0ae8de03:44343 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/3d7d90a0-d2fc-ab12-ff81-43a91f4072b2/oldWALs
2024-11-17T20:38:16,845 INFO  [RS:0;0b0b0ae8de03:44343 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 0b0b0ae8de03%2C44343%2C1731875895243.meta:.meta(num 1731875896390)
2024-11-17T20:38:16,846 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:38:16,846 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:38:16,846 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:38:16,846 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:38:16,846 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:38:16,848 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33139 is added to blk_1073741833_1009 (size=93)
2024-11-17T20:38:16,848 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45183 is added to blk_1073741833_1009 (size=93)
2024-11-17T20:38:16,848 INFO  [regionserver/0b0b0ae8de03:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-11-17T20:38:16,848 INFO  [regionserver/0b0b0ae8de03:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-11-17T20:38:16,850 DEBUG [RS:0;0b0b0ae8de03:44343 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/3d7d90a0-d2fc-ab12-ff81-43a91f4072b2/oldWALs
2024-11-17T20:38:16,850 INFO  [RS:0;0b0b0ae8de03:44343 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 0b0b0ae8de03%2C44343%2C1731875895243:(num 1731875895972)
2024-11-17T20:38:16,850 DEBUG [RS:0;0b0b0ae8de03:44343 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T20:38:16,850 INFO  [RS:0;0b0b0ae8de03:44343 {}] regionserver.LeaseManager(133): Closed leases
2024-11-17T20:38:16,850 INFO  [RS:0;0b0b0ae8de03:44343 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-17T20:38:16,850 INFO  [RS:0;0b0b0ae8de03:44343 {}] hbase.ChoreService(370): Chore service for: regionserver/0b0b0ae8de03:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-17T20:38:16,850 INFO  [RS:0;0b0b0ae8de03:44343 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-17T20:38:16,850 INFO  [regionserver/0b0b0ae8de03:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-17T20:38:16,850 INFO  [RS:0;0b0b0ae8de03:44343 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44343
2024-11-17T20:38:16,859 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,37509,1731875709020/0b0b0ae8de03%2C37509%2C1731875709020.1731875709295
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:38:16,860 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44343-0x1014a84f1ff0001, quorum=127.0.0.1:52912, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/0b0b0ae8de03,44343,1731875895243
2024-11-17T20:38:16,860 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42011-0x1014a84f1ff0000, quorum=127.0.0.1:52912, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-17T20:38:16,860 INFO  [RS:0;0b0b0ae8de03:44343 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-17T20:38:16,862 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/WALs/0b0b0ae8de03,33133,1731875707434/0b0b0ae8de03%2C33133%2C1731875707434.meta.1731875708822.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:38:16,871 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [0b0b0ae8de03,44343,1731875895243]
2024-11-17T20:38:16,881 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/0b0b0ae8de03,44343,1731875895243 already deleted, retry=false
2024-11-17T20:38:16,881 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 0b0b0ae8de03,44343,1731875895243 expired; onlineServers=0
2024-11-17T20:38:16,881 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '0b0b0ae8de03,42011,1731875894882' *****
2024-11-17T20:38:16,881 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-17T20:38:16,881 INFO  [M:0;0b0b0ae8de03:42011 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-17T20:38:16,881 INFO  [M:0;0b0b0ae8de03:42011 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-17T20:38:16,881 DEBUG [M:0;0b0b0ae8de03:42011 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-17T20:38:16,881 DEBUG [M:0;0b0b0ae8de03:42011 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-17T20:38:16,881 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-17T20:38:16,881 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster-HFileCleaner.large.0-1731875895685 {}] cleaner.HFileCleaner(306): Exit Thread[master/0b0b0ae8de03:0:becomeActiveMaster-HFileCleaner.large.0-1731875895685,5,FailOnTimeoutGroup]
2024-11-17T20:38:16,881 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster-HFileCleaner.small.0-1731875895685 {}] cleaner.HFileCleaner(306): Exit Thread[master/0b0b0ae8de03:0:becomeActiveMaster-HFileCleaner.small.0-1731875895685,5,FailOnTimeoutGroup]
2024-11-17T20:38:16,882 INFO  [M:0;0b0b0ae8de03:42011 {}] hbase.ChoreService(370): Chore service for: master/0b0b0ae8de03:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-17T20:38:16,882 INFO  [M:0;0b0b0ae8de03:42011 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-17T20:38:16,882 DEBUG [M:0;0b0b0ae8de03:42011 {}] master.HMaster(1795): Stopping service threads
2024-11-17T20:38:16,882 INFO  [M:0;0b0b0ae8de03:42011 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-17T20:38:16,882 INFO  [M:0;0b0b0ae8de03:42011 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-17T20:38:16,882 INFO  [M:0;0b0b0ae8de03:42011 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-17T20:38:16,882 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-17T20:38:16,892 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42011-0x1014a84f1ff0000, quorum=127.0.0.1:52912, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-17T20:38:16,892 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42011-0x1014a84f1ff0000, quorum=127.0.0.1:52912, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T20:38:16,892 DEBUG [M:0;0b0b0ae8de03:42011 {}] zookeeper.ZKUtil(347): master:42011-0x1014a84f1ff0000, quorum=127.0.0.1:52912, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-17T20:38:16,892 WARN  [M:0;0b0b0ae8de03:42011 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-17T20:38:16,893 INFO  [M:0;0b0b0ae8de03:42011 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:38271/user/jenkins/test-data/3d7d90a0-d2fc-ab12-ff81-43a91f4072b2/.lastflushedseqids
2024-11-17T20:38:16,900 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45183 is added to blk_1073741838_1014 (size=99)
2024-11-17T20:38:16,900 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33139 is added to blk_1073741838_1014 (size=99)
2024-11-17T20:38:16,901 INFO  [M:0;0b0b0ae8de03:42011 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-17T20:38:16,901 INFO  [M:0;0b0b0ae8de03:42011 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-17T20:38:16,901 DEBUG [M:0;0b0b0ae8de03:42011 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-17T20:38:16,901 INFO  [M:0;0b0b0ae8de03:42011 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T20:38:16,901 DEBUG [M:0;0b0b0ae8de03:42011 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T20:38:16,901 DEBUG [M:0;0b0b0ae8de03:42011 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-17T20:38:16,901 DEBUG [M:0;0b0b0ae8de03:42011 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T20:38:16,901 INFO  [M:0;0b0b0ae8de03:42011 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=7.67 KB heapSize=11.34 KB
2024-11-17T20:38:16,915 DEBUG [M:0;0b0b0ae8de03:42011 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38271/user/jenkins/test-data/3d7d90a0-d2fc-ab12-ff81-43a91f4072b2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/43bb301b35fc4e4bb16adb7b5e6f1677 is 82, key is hbase:meta,,1/info:regioninfo/1731875896420/Put/seqid=0
2024-11-17T20:38:16,919 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45183 is added to blk_1073741839_1015 (size=5672)
2024-11-17T20:38:16,919 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33139 is added to blk_1073741839_1015 (size=5672)
2024-11-17T20:38:16,920 INFO  [M:0;0b0b0ae8de03:42011 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:38271/user/jenkins/test-data/3d7d90a0-d2fc-ab12-ff81-43a91f4072b2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/43bb301b35fc4e4bb16adb7b5e6f1677
2024-11-17T20:38:16,939 DEBUG [M:0;0b0b0ae8de03:42011 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38271/user/jenkins/test-data/3d7d90a0-d2fc-ab12-ff81-43a91f4072b2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/30a66962d9104b58ba92e20b1d2a3564 is 240, key is \x00\x00\x00\x00\x00\x00\x00\x01/proc:d/1731875896475/Put/seqid=0
2024-11-17T20:38:16,943 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33139 is added to blk_1073741840_1016 (size=5275)
2024-11-17T20:38:16,943 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45183 is added to blk_1073741840_1016 (size=5275)
2024-11-17T20:38:16,943 INFO  [M:0;0b0b0ae8de03:42011 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.06 KB at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:38271/user/jenkins/test-data/3d7d90a0-d2fc-ab12-ff81-43a91f4072b2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/30a66962d9104b58ba92e20b1d2a3564
2024-11-17T20:38:16,961 DEBUG [M:0;0b0b0ae8de03:42011 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38271/user/jenkins/test-data/3d7d90a0-d2fc-ab12-ff81-43a91f4072b2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/594b332093174de2827e30bbf3d4b985 is 69, key is 0b0b0ae8de03,44343,1731875895243/rs:state/1731875895808/Put/seqid=0
2024-11-17T20:38:16,965 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33139 is added to blk_1073741841_1017 (size=5156)
2024-11-17T20:38:16,965 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45183 is added to blk_1073741841_1017 (size=5156)
2024-11-17T20:38:16,965 INFO  [M:0;0b0b0ae8de03:42011 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:38271/user/jenkins/test-data/3d7d90a0-d2fc-ab12-ff81-43a91f4072b2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/594b332093174de2827e30bbf3d4b985
2024-11-17T20:38:16,971 INFO  [RS:0;0b0b0ae8de03:44343 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-17T20:38:16,971 INFO  [RS:0;0b0b0ae8de03:44343 {}] regionserver.HRegionServer(1031): Exiting; stopping=0b0b0ae8de03,44343,1731875895243; zookeeper connection closed.
2024-11-17T20:38:16,971 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44343-0x1014a84f1ff0001, quorum=127.0.0.1:52912, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-17T20:38:16,971 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44343-0x1014a84f1ff0001, quorum=127.0.0.1:52912, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-17T20:38:16,971 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@536204f3 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@536204f3
2024-11-17T20:38:16,971 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-17T20:38:16,983 DEBUG [M:0;0b0b0ae8de03:42011 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38271/user/jenkins/test-data/3d7d90a0-d2fc-ab12-ff81-43a91f4072b2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/16f24689e1ea4ee6af20c663f47b0ee3 is 52, key is load_balancer_on/state:d/1731875896584/Put/seqid=0
2024-11-17T20:38:16,987 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45183 is added to blk_1073741842_1018 (size=5056)
2024-11-17T20:38:16,987 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33139 is added to blk_1073741842_1018 (size=5056)
2024-11-17T20:38:16,987 INFO  [M:0;0b0b0ae8de03:42011 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:38271/user/jenkins/test-data/3d7d90a0-d2fc-ab12-ff81-43a91f4072b2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/16f24689e1ea4ee6af20c663f47b0ee3
2024-11-17T20:38:16,991 DEBUG [M:0;0b0b0ae8de03:42011 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38271/user/jenkins/test-data/3d7d90a0-d2fc-ab12-ff81-43a91f4072b2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/43bb301b35fc4e4bb16adb7b5e6f1677 as hdfs://localhost:38271/user/jenkins/test-data/3d7d90a0-d2fc-ab12-ff81-43a91f4072b2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/43bb301b35fc4e4bb16adb7b5e6f1677
2024-11-17T20:38:16,995 INFO  [M:0;0b0b0ae8de03:42011 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38271/user/jenkins/test-data/3d7d90a0-d2fc-ab12-ff81-43a91f4072b2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/43bb301b35fc4e4bb16adb7b5e6f1677, entries=8, sequenceid=29, filesize=5.5 K
2024-11-17T20:38:16,996 DEBUG [M:0;0b0b0ae8de03:42011 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38271/user/jenkins/test-data/3d7d90a0-d2fc-ab12-ff81-43a91f4072b2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/30a66962d9104b58ba92e20b1d2a3564 as hdfs://localhost:38271/user/jenkins/test-data/3d7d90a0-d2fc-ab12-ff81-43a91f4072b2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/30a66962d9104b58ba92e20b1d2a3564
2024-11-17T20:38:17,000 INFO  [M:0;0b0b0ae8de03:42011 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38271/user/jenkins/test-data/3d7d90a0-d2fc-ab12-ff81-43a91f4072b2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/30a66962d9104b58ba92e20b1d2a3564, entries=3, sequenceid=29, filesize=5.2 K
2024-11-17T20:38:17,001 DEBUG [M:0;0b0b0ae8de03:42011 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38271/user/jenkins/test-data/3d7d90a0-d2fc-ab12-ff81-43a91f4072b2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/594b332093174de2827e30bbf3d4b985 as hdfs://localhost:38271/user/jenkins/test-data/3d7d90a0-d2fc-ab12-ff81-43a91f4072b2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/594b332093174de2827e30bbf3d4b985
2024-11-17T20:38:17,005 INFO  [M:0;0b0b0ae8de03:42011 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38271/user/jenkins/test-data/3d7d90a0-d2fc-ab12-ff81-43a91f4072b2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/594b332093174de2827e30bbf3d4b985, entries=1, sequenceid=29, filesize=5.0 K
2024-11-17T20:38:17,006 DEBUG [M:0;0b0b0ae8de03:42011 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38271/user/jenkins/test-data/3d7d90a0-d2fc-ab12-ff81-43a91f4072b2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/16f24689e1ea4ee6af20c663f47b0ee3 as hdfs://localhost:38271/user/jenkins/test-data/3d7d90a0-d2fc-ab12-ff81-43a91f4072b2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/16f24689e1ea4ee6af20c663f47b0ee3
2024-11-17T20:38:17,010 INFO  [M:0;0b0b0ae8de03:42011 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38271/user/jenkins/test-data/3d7d90a0-d2fc-ab12-ff81-43a91f4072b2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/16f24689e1ea4ee6af20c663f47b0ee3, entries=1, sequenceid=29, filesize=4.9 K
2024-11-17T20:38:17,011 INFO  [M:0;0b0b0ae8de03:42011 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 110ms, sequenceid=29, compaction requested=false
2024-11-17T20:38:17,012 INFO  [M:0;0b0b0ae8de03:42011 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T20:38:17,012 DEBUG [M:0;0b0b0ae8de03:42011 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731875896901Disabling compacts and flushes for region at 1731875896901Disabling writes for close at 1731875896901Obtaining lock to block concurrent updates at 1731875896901Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731875896901Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=7850, getHeapSize=11544, getOffHeapSize=0, getCellsCount=36 at 1731875896902 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731875896903 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731875896903Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731875896915 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731875896915Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731875896923 (+8 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731875896938 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731875896938Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731875896947 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731875896960 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731875896960Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731875896969 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731875896982 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731875896982Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@77bee90a: reopening flushed file at 1731875896991 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@8f8c5c6: reopening flushed file at 1731875896995 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@18cdc877: reopening flushed file at 1731875897000 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@17f0fd3d: reopening flushed file at 1731875897005 (+5 ms)Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 110ms, sequenceid=29, compaction requested=false at 1731875897011 (+6 ms)Writing region close event to WAL at 1731875897012 (+1 ms)Closed at 1731875897012
2024-11-17T20:38:17,012 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:38:17,012 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:38:17,012 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:38:17,012 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:38:17,012 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T20:38:17,014 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45183 is added to blk_1073741830_1006 (size=10311)
2024-11-17T20:38:17,014 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33139 is added to blk_1073741830_1006 (size=10311)
2024-11-17T20:38:17,015 INFO  [M:0;0b0b0ae8de03:42011 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-17T20:38:17,015 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-17T20:38:17,015 INFO  [M:0;0b0b0ae8de03:42011 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42011
2024-11-17T20:38:17,015 INFO  [M:0;0b0b0ae8de03:42011 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-17T20:38:17,027 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33551/user/jenkins/test-data/66ed7fad-7246-2283-f933-7571ff172899/MasterData/WALs/0b0b0ae8de03,40559,1731875707247/0b0b0ae8de03%2C40559%2C1731875707247.1731875707650
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T20:38:17,111 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:38:17,111 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:38:17,112 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:38:17,112 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:38:17,112 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:38:17,113 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:38:17,114 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:38:17,115 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:38:17,124 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42011-0x1014a84f1ff0000, quorum=127.0.0.1:52912, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-17T20:38:17,124 INFO  [M:0;0b0b0ae8de03:42011 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-17T20:38:17,124 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42011-0x1014a84f1ff0000, quorum=127.0.0.1:52912, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-17T20:38:17,126 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3565f430{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T20:38:17,126 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@16ced564{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-17T20:38:17,126 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-17T20:38:17,126 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@59a6b271{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-17T20:38:17,126 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6c2fc043{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fdc09039-3224-6f0d-ffe0-8e19966578de/hadoop.log.dir/,STOPPED}
2024-11-17T20:38:17,144 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:38:17,144 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:38:17,144 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:38:17,144 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:38:17,144 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:38:17,145 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:38:17,150 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:38:17,150 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:38:17,150 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:38:17,153 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T20:38:17,156 WARN  [BP-1884258858-172.17.0.2-1731875892739 heartbeating to localhost/127.0.0.1:38271 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-17T20:38:17,156 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-17T20:38:17,156 WARN  [BP-1884258858-172.17.0.2-1731875892739 heartbeating to localhost/127.0.0.1:38271 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1884258858-172.17.0.2-1731875892739 (Datanode Uuid 328f7c29-1fee-4879-932b-76574a90d7bc) service to localhost/127.0.0.1:38271
2024-11-17T20:38:17,156 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-17T20:38:17,156 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fdc09039-3224-6f0d-ffe0-8e19966578de/cluster_0a835a85-a168-a15e-c665-c668c56cd27d/data/data3/current/BP-1884258858-172.17.0.2-1731875892739 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T20:38:17,157 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fdc09039-3224-6f0d-ffe0-8e19966578de/cluster_0a835a85-a168-a15e-c665-c668c56cd27d/data/data4/current/BP-1884258858-172.17.0.2-1731875892739 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T20:38:17,157 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-17T20:38:17,159 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@64440bf6{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T20:38:17,159 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2551ca75{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-17T20:38:17,159 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-17T20:38:17,159 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5f638c2a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-17T20:38:17,159 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2fb7753b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fdc09039-3224-6f0d-ffe0-8e19966578de/hadoop.log.dir/,STOPPED}
2024-11-17T20:38:17,160 WARN  [BP-1884258858-172.17.0.2-1731875892739 heartbeating to localhost/127.0.0.1:38271 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-17T20:38:17,160 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-17T20:38:17,160 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-17T20:38:17,160 WARN  [BP-1884258858-172.17.0.2-1731875892739 heartbeating to localhost/127.0.0.1:38271 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1884258858-172.17.0.2-1731875892739 (Datanode Uuid b8e80017-b64e-4d84-9a5d-ddd073b2590a) service to localhost/127.0.0.1:38271
2024-11-17T20:38:17,160 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fdc09039-3224-6f0d-ffe0-8e19966578de/cluster_0a835a85-a168-a15e-c665-c668c56cd27d/data/data1/current/BP-1884258858-172.17.0.2-1731875892739 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T20:38:17,161 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fdc09039-3224-6f0d-ffe0-8e19966578de/cluster_0a835a85-a168-a15e-c665-c668c56cd27d/data/data2/current/BP-1884258858-172.17.0.2-1731875892739 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T20:38:17,161 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-17T20:38:17,165 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@12ac7c6{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-17T20:38:17,165 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2b13a29{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-17T20:38:17,165 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-17T20:38:17,166 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4c4c959a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-17T20:38:17,166 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3b55cb8{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fdc09039-3224-6f0d-ffe0-8e19966578de/hadoop.log.dir/,STOPPED}
2024-11-17T20:38:17,170 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-17T20:38:17,183 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-17T20:38:17,192 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnNothingWritten Thread=271 (was 233)
Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:38271 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-44-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-42-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:38271 from jenkins.hfs.7
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-43-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:38271
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-43-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:38271
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.7@localhost:38271
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-42-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-45-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-43-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-16-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-42-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:38271
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-45-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-45-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-44-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-44-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:38271 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: HMaster-EventLoopGroup-16-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-16-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:38271
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=536 (was 521) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=234 (was 228) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8225 (was 8233)