2024-12-11 16:21:50,240 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca
2024-12-11 16:21:50,256 main DEBUG Took 0.013979 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging
2024-12-11 16:21:50,256 main DEBUG PluginManager 'Core' found 129 plugins
2024-12-11 16:21:50,257 main DEBUG PluginManager 'Level' found 0 plugins
2024-12-11 16:21:50,258 main DEBUG PluginManager 'Lookup' found 16 plugins
2024-12-11 16:21:50,260 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-11 16:21:50,268 main DEBUG PluginManager 'TypeConverter' found 26 plugins
2024-12-11 16:21:50,285 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-11 16:21:50,287 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-11 16:21:50,288 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-11 16:21:50,288 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-11 16:21:50,289 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-11 16:21:50,289 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-11 16:21:50,290 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-11 16:21:50,291 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-11 16:21:50,291 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-11 16:21:50,292 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-11 16:21:50,293 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-11 16:21:50,293 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-11 16:21:50,294 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-11 16:21:50,295 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-11 16:21:50,295 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-11 16:21:50,296 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-11 16:21:50,296 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-11 16:21:50,297 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-11 16:21:50,297 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-11 16:21:50,298 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-11 16:21:50,298 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-11 16:21:50,299 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-11 16:21:50,299 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-11 16:21:50,300 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-11 16:21:50,300 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-11 16:21:50,301 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger].
2024-12-11 16:21:50,303 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-11 16:21:50,304 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin].
2024-12-11 16:21:50,307 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root})
2024-12-11 16:21:50,307 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout].
2024-12-11 16:21:50,309 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null")
2024-12-11 16:21:50,309 main DEBUG PluginManager 'Converter' found 47 plugins
2024-12-11 16:21:50,320 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender].
2024-12-11 16:21:50,324 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={})
2024-12-11 16:21:50,326 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR
2024-12-11 16:21:50,327 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin].
2024-12-11 16:21:50,327 main DEBUG createAppenders(={Console})
2024-12-11 16:21:50,328 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca initialized
2024-12-11 16:21:50,329 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca
2024-12-11 16:21:50,329 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca OK.
2024-12-11 16:21:50,330 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1
2024-12-11 16:21:50,330 main DEBUG OutputStream closed
2024-12-11 16:21:50,330 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true
2024-12-11 16:21:50,330 main DEBUG Appender DefaultConsole-1 stopped with status true
2024-12-11 16:21:50,331 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@6404f418 OK
2024-12-11 16:21:50,448 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6
2024-12-11 16:21:50,450 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger
2024-12-11 16:21:50,451 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector
2024-12-11 16:21:50,452 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=
2024-12-11 16:21:50,453 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory
2024-12-11 16:21:50,453 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter
2024-12-11 16:21:50,454 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper
2024-12-11 16:21:50,454 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j
2024-12-11 16:21:50,455 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl
2024-12-11 16:21:50,455 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans
2024-12-11 16:21:50,455 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase
2024-12-11 16:21:50,456 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop
2024-12-11 16:21:50,456 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers
2024-12-11 16:21:50,456 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices
2024-12-11 16:21:50,456 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig
2024-12-11 16:21:50,457 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel
2024-12-11 16:21:50,457 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore
2024-12-11 16:21:50,458 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console
2024-12-11 16:21:50,459 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-12-11 16:21:50,460 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@6dab9b6d) with optional ClassLoader: null
2024-12-11 16:21:50,460 main DEBUG Shutdown hook enabled. Registering a new one.
2024-12-11 16:21:50,461 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@6dab9b6d] started OK.
2024-12-11T16:21:50,714 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/762be7fd-a69f-281d-5246-84312cc22b69
2024-12-11 16:21:50,717 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED)
2024-12-11 16:21:50,717 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-12-11T16:21:50,726 INFO  [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestLogRolling timeout: 13 mins
2024-12-11T16:21:50,758 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testSlowSyncLogRolling Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=1185, ProcessCount=11, AvailableMemoryMB=2027
2024-12-11T16:21:50,761 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-12-11T16:21:50,779 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/762be7fd-a69f-281d-5246-84312cc22b69/cluster_8f17652b-4025-2fa7-ad27-5fb00f5849ad, deleteOnExit=true
2024-12-11T16:21:50,779 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-12-11T16:21:50,781 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/762be7fd-a69f-281d-5246-84312cc22b69/test.cache.data in system properties and HBase conf
2024-12-11T16:21:50,782 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/762be7fd-a69f-281d-5246-84312cc22b69/hadoop.tmp.dir in system properties and HBase conf
2024-12-11T16:21:50,782 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/762be7fd-a69f-281d-5246-84312cc22b69/hadoop.log.dir in system properties and HBase conf
2024-12-11T16:21:50,783 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/762be7fd-a69f-281d-5246-84312cc22b69/mapreduce.cluster.local.dir in system properties and HBase conf
2024-12-11T16:21:50,784 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/762be7fd-a69f-281d-5246-84312cc22b69/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-12-11T16:21:50,784 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-12-11T16:21:50,900 WARN  [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
2024-12-11T16:21:51,010 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-11T16:21:51,015 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/762be7fd-a69f-281d-5246-84312cc22b69/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-12-11T16:21:51,016 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/762be7fd-a69f-281d-5246-84312cc22b69/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-12-11T16:21:51,016 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/762be7fd-a69f-281d-5246-84312cc22b69/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-12-11T16:21:51,017 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/762be7fd-a69f-281d-5246-84312cc22b69/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-11T16:21:51,018 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/762be7fd-a69f-281d-5246-84312cc22b69/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-12-11T16:21:51,019 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/762be7fd-a69f-281d-5246-84312cc22b69/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-12-11T16:21:51,019 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/762be7fd-a69f-281d-5246-84312cc22b69/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-11T16:21:51,020 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/762be7fd-a69f-281d-5246-84312cc22b69/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-11T16:21:51,021 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/762be7fd-a69f-281d-5246-84312cc22b69/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-12-11T16:21:51,021 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/762be7fd-a69f-281d-5246-84312cc22b69/nfs.dump.dir in system properties and HBase conf
2024-12-11T16:21:51,022 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/762be7fd-a69f-281d-5246-84312cc22b69/java.io.tmpdir in system properties and HBase conf
2024-12-11T16:21:51,022 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/762be7fd-a69f-281d-5246-84312cc22b69/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-11T16:21:51,023 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/762be7fd-a69f-281d-5246-84312cc22b69/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-12-11T16:21:51,024 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/762be7fd-a69f-281d-5246-84312cc22b69/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-12-11T16:21:51,618 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-11T16:21:52,376 WARN  [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
2024-12-11T16:21:52,454 INFO  [Time-limited test {}] log.Log(170): Logging initialized @3124ms to org.eclipse.jetty.util.log.Slf4jLog
2024-12-11T16:21:52,524 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-11T16:21:52,593 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-11T16:21:52,624 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-11T16:21:52,624 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-11T16:21:52,626 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-11T16:21:52,647 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-11T16:21:52,657 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3aee6cb7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/762be7fd-a69f-281d-5246-84312cc22b69/hadoop.log.dir/,AVAILABLE}
2024-12-11T16:21:52,658 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@380b8195{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-11T16:21:52,961 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6de997b9{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/762be7fd-a69f-281d-5246-84312cc22b69/java.io.tmpdir/jetty-localhost-43565-hadoop-hdfs-3_4_1-tests_jar-_-any-17089861783335793520/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-11T16:21:52,972 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7a0da00a{HTTP/1.1, (http/1.1)}{localhost:43565}
2024-12-11T16:21:52,973 INFO  [Time-limited test {}] server.Server(415): Started @3643ms
2024-12-11T16:21:53,027 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-11T16:21:53,698 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-11T16:21:53,709 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-11T16:21:53,710 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-11T16:21:53,710 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-11T16:21:53,711 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-11T16:21:53,711 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@26c88bf4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/762be7fd-a69f-281d-5246-84312cc22b69/hadoop.log.dir/,AVAILABLE}
2024-12-11T16:21:53,712 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@621a7cbc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-11T16:21:53,861 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@11e88411{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/762be7fd-a69f-281d-5246-84312cc22b69/java.io.tmpdir/jetty-localhost-41445-hadoop-hdfs-3_4_1-tests_jar-_-any-3993960441475135742/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T16:21:53,863 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@75423500{HTTP/1.1, (http/1.1)}{localhost:41445}
2024-12-11T16:21:53,864 INFO  [Time-limited test {}] server.Server(415): Started @4534ms
2024-12-11T16:21:53,937 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-11T16:21:54,165 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-11T16:21:54,192 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-11T16:21:54,224 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-11T16:21:54,224 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-11T16:21:54,225 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-11T16:21:54,233 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@43794ae7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/762be7fd-a69f-281d-5246-84312cc22b69/hadoop.log.dir/,AVAILABLE}
2024-12-11T16:21:54,234 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@35f1cf70{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-11T16:21:54,398 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@590b36b7{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/762be7fd-a69f-281d-5246-84312cc22b69/java.io.tmpdir/jetty-localhost-34311-hadoop-hdfs-3_4_1-tests_jar-_-any-136683126770686390/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T16:21:54,399 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@237fc06a{HTTP/1.1, (http/1.1)}{localhost:34311}
2024-12-11T16:21:54,400 INFO  [Time-limited test {}] server.Server(415): Started @5070ms
2024-12-11T16:21:54,403 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-11T16:21:55,437 WARN  [Thread-97 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/762be7fd-a69f-281d-5246-84312cc22b69/cluster_8f17652b-4025-2fa7-ad27-5fb00f5849ad/data/data2/current/BP-1818873448-172.17.0.2-1733934111750/current, will proceed with Du for space computation calculation, 
2024-12-11T16:21:55,437 WARN  [Thread-96 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/762be7fd-a69f-281d-5246-84312cc22b69/cluster_8f17652b-4025-2fa7-ad27-5fb00f5849ad/data/data1/current/BP-1818873448-172.17.0.2-1733934111750/current, will proceed with Du for space computation calculation, 
2024-12-11T16:21:55,468 WARN  [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-11T16:21:55,483 WARN  [Thread-107 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/762be7fd-a69f-281d-5246-84312cc22b69/cluster_8f17652b-4025-2fa7-ad27-5fb00f5849ad/data/data3/current/BP-1818873448-172.17.0.2-1733934111750/current, will proceed with Du for space computation calculation, 
2024-12-11T16:21:55,485 WARN  [Thread-108 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/762be7fd-a69f-281d-5246-84312cc22b69/cluster_8f17652b-4025-2fa7-ad27-5fb00f5849ad/data/data4/current/BP-1818873448-172.17.0.2-1733934111750/current, will proceed with Du for space computation calculation, 
2024-12-11T16:21:55,528 WARN  [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-11T16:21:55,535 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xbb76e78c3d7ca29f with lease ID 0x974e271c41d15f13: Processing first storage report for DS-0137187c-2bdf-42ef-886c-d633906df312 from datanode DatanodeRegistration(127.0.0.1:42931, datanodeUuid=8ba32a4e-b51a-4d2f-8424-e054183c8047, infoPort=41771, infoSecurePort=0, ipcPort=41789, storageInfo=lv=-57;cid=testClusterID;nsid=1947243362;c=1733934111750)
2024-12-11T16:21:55,536 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbb76e78c3d7ca29f with lease ID 0x974e271c41d15f13: from storage DS-0137187c-2bdf-42ef-886c-d633906df312 node DatanodeRegistration(127.0.0.1:42931, datanodeUuid=8ba32a4e-b51a-4d2f-8424-e054183c8047, infoPort=41771, infoSecurePort=0, ipcPort=41789, storageInfo=lv=-57;cid=testClusterID;nsid=1947243362;c=1733934111750), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0
2024-12-11T16:21:55,537 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xbb76e78c3d7ca29f with lease ID 0x974e271c41d15f13: Processing first storage report for DS-65dce0c3-3409-4f4c-91be-d60d3ab9d4b4 from datanode DatanodeRegistration(127.0.0.1:42931, datanodeUuid=8ba32a4e-b51a-4d2f-8424-e054183c8047, infoPort=41771, infoSecurePort=0, ipcPort=41789, storageInfo=lv=-57;cid=testClusterID;nsid=1947243362;c=1733934111750)
2024-12-11T16:21:55,537 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbb76e78c3d7ca29f with lease ID 0x974e271c41d15f13: from storage DS-65dce0c3-3409-4f4c-91be-d60d3ab9d4b4 node DatanodeRegistration(127.0.0.1:42931, datanodeUuid=8ba32a4e-b51a-4d2f-8424-e054183c8047, infoPort=41771, infoSecurePort=0, ipcPort=41789, storageInfo=lv=-57;cid=testClusterID;nsid=1947243362;c=1733934111750), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-11T16:21:55,539 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x47d6d8df2dea86a8 with lease ID 0x974e271c41d15f14: Processing first storage report for DS-ae50a1ed-d15b-4f7e-b088-c3df73ccfe60 from datanode DatanodeRegistration(127.0.0.1:42705, datanodeUuid=257da275-4cb7-4478-9896-000aa2c9722a, infoPort=46071, infoSecurePort=0, ipcPort=33921, storageInfo=lv=-57;cid=testClusterID;nsid=1947243362;c=1733934111750)
2024-12-11T16:21:55,540 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x47d6d8df2dea86a8 with lease ID 0x974e271c41d15f14: from storage DS-ae50a1ed-d15b-4f7e-b088-c3df73ccfe60 node DatanodeRegistration(127.0.0.1:42705, datanodeUuid=257da275-4cb7-4478-9896-000aa2c9722a, infoPort=46071, infoSecurePort=0, ipcPort=33921, storageInfo=lv=-57;cid=testClusterID;nsid=1947243362;c=1733934111750), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-11T16:21:55,540 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x47d6d8df2dea86a8 with lease ID 0x974e271c41d15f14: Processing first storage report for DS-61d60c6a-4d15-4e5d-917e-9e65dd920ba8 from datanode DatanodeRegistration(127.0.0.1:42705, datanodeUuid=257da275-4cb7-4478-9896-000aa2c9722a, infoPort=46071, infoSecurePort=0, ipcPort=33921, storageInfo=lv=-57;cid=testClusterID;nsid=1947243362;c=1733934111750)
2024-12-11T16:21:55,540 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x47d6d8df2dea86a8 with lease ID 0x974e271c41d15f14: from storage DS-61d60c6a-4d15-4e5d-917e-9e65dd920ba8 node DatanodeRegistration(127.0.0.1:42705, datanodeUuid=257da275-4cb7-4478-9896-000aa2c9722a, infoPort=46071, infoSecurePort=0, ipcPort=33921, storageInfo=lv=-57;cid=testClusterID;nsid=1947243362;c=1733934111750), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-11T16:21:55,618 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/762be7fd-a69f-281d-5246-84312cc22b69
2024-12-11T16:21:55,731 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/762be7fd-a69f-281d-5246-84312cc22b69/cluster_8f17652b-4025-2fa7-ad27-5fb00f5849ad/zookeeper_0, clientPort=53474, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/762be7fd-a69f-281d-5246-84312cc22b69/cluster_8f17652b-4025-2fa7-ad27-5fb00f5849ad/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/762be7fd-a69f-281d-5246-84312cc22b69/cluster_8f17652b-4025-2fa7-ad27-5fb00f5849ad/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-11T16:21:55,744 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=53474
2024-12-11T16:21:55,758 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T16:21:55,761 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T16:21:56,072 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42705 is added to blk_1073741825_1001 (size=7)
2024-12-11T16:21:56,073 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42931 is added to blk_1073741825_1001 (size=7)
2024-12-11T16:21:56,487 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c with version=8
2024-12-11T16:21:56,488 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/hbase-staging
2024-12-11T16:21:56,587 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16
2024-12-11T16:21:56,887 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/7446a65196a1:0 server-side Connection retries=45
2024-12-11T16:21:56,900 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-11T16:21:56,901 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-11T16:21:56,907 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-11T16:21:56,907 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-11T16:21:56,907 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-11T16:21:57,109 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-12-11T16:21:57,199 INFO  [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl
2024-12-11T16:21:57,211 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout
2024-12-11T16:21:57,216 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-11T16:21:57,248 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 15486 (auto-detected)
2024-12-11T16:21:57,249 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected)
2024-12-11T16:21:57,274 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39377
2024-12-11T16:21:57,305 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:39377 connecting to ZooKeeper ensemble=127.0.0.1:53474
2024-12-11T16:21:57,396 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:393770x0, quorum=127.0.0.1:53474, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-11T16:21:57,400 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:39377-0x10015bcaa160000 connected
2024-12-11T16:21:57,586 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T16:21:57,590 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T16:21:57,612 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39377-0x10015bcaa160000, quorum=127.0.0.1:53474, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-11T16:21:57,618 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c, hbase.cluster.distributed=false
2024-12-11T16:21:57,646 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39377-0x10015bcaa160000, quorum=127.0.0.1:53474, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-11T16:21:57,652 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39377
2024-12-11T16:21:57,656 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39377
2024-12-11T16:21:57,671 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39377
2024-12-11T16:21:57,679 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39377
2024-12-11T16:21:57,681 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39377
2024-12-11T16:21:57,871 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/7446a65196a1:0 server-side Connection retries=45
2024-12-11T16:21:57,874 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-11T16:21:57,874 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-11T16:21:57,874 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-11T16:21:57,875 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-11T16:21:57,875 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-11T16:21:57,879 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-11T16:21:57,883 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-11T16:21:57,891 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43823
2024-12-11T16:21:57,894 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43823 connecting to ZooKeeper ensemble=127.0.0.1:53474
2024-12-11T16:21:57,896 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T16:21:57,901 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T16:21:57,931 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:438230x0, quorum=127.0.0.1:53474, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-11T16:21:57,935 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43823-0x10015bcaa160001 connected
2024-12-11T16:21:57,940 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43823-0x10015bcaa160001, quorum=127.0.0.1:53474, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-11T16:21:57,947 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-11T16:21:57,963 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-11T16:21:57,979 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43823-0x10015bcaa160001, quorum=127.0.0.1:53474, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-11T16:21:58,008 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43823-0x10015bcaa160001, quorum=127.0.0.1:53474, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-11T16:21:58,015 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43823
2024-12-11T16:21:58,019 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43823
2024-12-11T16:21:58,027 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43823
2024-12-11T16:21:58,038 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43823
2024-12-11T16:21:58,039 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43823
2024-12-11T16:21:58,074 DEBUG [M:0;7446a65196a1:39377 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;7446a65196a1:39377
2024-12-11T16:21:58,080 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/7446a65196a1,39377,1733934116649
2024-12-11T16:21:58,100 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39377-0x10015bcaa160000, quorum=127.0.0.1:53474, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-11T16:21:58,100 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43823-0x10015bcaa160001, quorum=127.0.0.1:53474, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-11T16:21:58,115 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39377-0x10015bcaa160000, quorum=127.0.0.1:53474, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/7446a65196a1,39377,1733934116649
2024-12-11T16:21:58,162 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39377-0x10015bcaa160000, quorum=127.0.0.1:53474, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:21:58,162 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43823-0x10015bcaa160001, quorum=127.0.0.1:53474, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-11T16:21:58,162 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43823-0x10015bcaa160001, quorum=127.0.0.1:53474, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:21:58,165 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39377-0x10015bcaa160000, quorum=127.0.0.1:53474, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-11T16:21:58,175 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/7446a65196a1,39377,1733934116649 from backup master directory
2024-12-11T16:21:58,193 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39377-0x10015bcaa160000, quorum=127.0.0.1:53474, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/7446a65196a1,39377,1733934116649
2024-12-11T16:21:58,194 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43823-0x10015bcaa160001, quorum=127.0.0.1:53474, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-11T16:21:58,194 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39377-0x10015bcaa160000, quorum=127.0.0.1:53474, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-11T16:21:58,195 WARN  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-11T16:21:58,196 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=7446a65196a1,39377,1733934116649
2024-12-11T16:21:58,197 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0
2024-12-11T16:21:58,199 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0
2024-12-11T16:21:58,290 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/hbase.id] with ID: 954ed961-52d2-4f35-a79a-e9949be00ee6
2024-12-11T16:21:58,290 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/.tmp/hbase.id
2024-12-11T16:21:58,324 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42931 is added to blk_1073741826_1002 (size=42)
2024-12-11T16:21:58,324 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42705 is added to blk_1073741826_1002 (size=42)
2024-12-11T16:21:58,326 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/.tmp/hbase.id]:[hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/hbase.id]
2024-12-11T16:21:58,416 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T16:21:58,422 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-12-11T16:21:58,443 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 19ms.
2024-12-11T16:21:58,462 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39377-0x10015bcaa160000, quorum=127.0.0.1:53474, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:21:58,462 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43823-0x10015bcaa160001, quorum=127.0.0.1:53474, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:21:58,493 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42705 is added to blk_1073741827_1003 (size=196)
2024-12-11T16:21:58,494 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42931 is added to blk_1073741827_1003 (size=196)
2024-12-11T16:21:58,513 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-11T16:21:58,515 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-11T16:21:58,523 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-11T16:21:58,563 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42705 is added to blk_1073741828_1004 (size=1189)
2024-12-11T16:21:58,564 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42931 is added to blk_1073741828_1004 (size=1189)
2024-12-11T16:21:58,590 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/MasterData/data/master/store
2024-12-11T16:21:58,626 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42931 is added to blk_1073741829_1005 (size=34)
2024-12-11T16:21:58,626 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42705 is added to blk_1073741829_1005 (size=34)
2024-12-11T16:21:58,632 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure.
2024-12-11T16:21:58,635 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T16:21:58,637 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-11T16:21:58,637 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T16:21:58,637 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T16:21:58,639 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-11T16:21:58,639 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T16:21:58,639 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T16:21:58,641 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733934118637Disabling compacts and flushes for region at 1733934118637Disabling writes for close at 1733934118639 (+2 ms)Writing region close event to WAL at 1733934118639Closed at 1733934118639
2024-12-11T16:21:58,644 WARN  [master/7446a65196a1:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/MasterData/data/master/store/.initializing
2024-12-11T16:21:58,644 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/MasterData/WALs/7446a65196a1,39377,1733934116649
2024-12-11T16:21:58,668 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7446a65196a1%2C39377%2C1733934116649, suffix=, logDir=hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/MasterData/WALs/7446a65196a1,39377,1733934116649, archiveDir=hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/MasterData/oldWALs, maxLogs=10
2024-12-11T16:21:58,678 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7446a65196a1%2C39377%2C1733934116649.1733934118673
2024-12-11T16:21:58,738 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/MasterData/WALs/7446a65196a1,39377,1733934116649/7446a65196a1%2C39377%2C1733934116649.1733934118673
2024-12-11T16:21:58,764 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46071:46071),(127.0.0.1/127.0.0.1:41771:41771)]
2024-12-11T16:21:58,768 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-11T16:21:58,769 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T16:21:58,774 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T16:21:58,775 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T16:21:58,829 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-11T16:21:58,858 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-11T16:21:58,862 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:21:58,865 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T16:21:58,866 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-11T16:21:58,870 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-11T16:21:58,870 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:21:58,872 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-11T16:21:58,872 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-11T16:21:58,882 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-11T16:21:58,882 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:21:58,884 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-11T16:21:58,884 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-11T16:21:58,889 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-11T16:21:58,889 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:21:58,891 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-11T16:21:58,892 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T16:21:58,900 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-11T16:21:58,901 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-11T16:21:58,910 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T16:21:58,911 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T16:21:58,922 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-11T16:21:58,929 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T16:21:58,939 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-11T16:21:58,943 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=842382, jitterRate=0.07114449143409729}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-11T16:21:58,952 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1733934118793Initializing all the Stores at 1733934118796 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733934118797 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733934118798 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733934118798Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733934118798Cleaning up temporary data from old regions at 1733934118912 (+114 ms)Region opened successfully at 1733934118952 (+40 ms)
2024-12-11T16:21:58,955 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-11T16:21:59,016 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2620099d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7446a65196a1/172.17.0.2:0
2024-12-11T16:21:59,064 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-12-11T16:21:59,079 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-11T16:21:59,080 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-11T16:21:59,084 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-11T16:21:59,086 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec
2024-12-11T16:21:59,093 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec
2024-12-11T16:21:59,093 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-11T16:21:59,136 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-11T16:21:59,153 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39377-0x10015bcaa160000, quorum=127.0.0.1:53474, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-11T16:21:59,165 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-12-11T16:21:59,168 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-11T16:21:59,173 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39377-0x10015bcaa160000, quorum=127.0.0.1:53474, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-11T16:21:59,181 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-12-11T16:21:59,185 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-11T16:21:59,195 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39377-0x10015bcaa160000, quorum=127.0.0.1:53474, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-11T16:21:59,211 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-12-11T16:21:59,218 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39377-0x10015bcaa160000, quorum=127.0.0.1:53474, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-11T16:21:59,228 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-12-11T16:21:59,256 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39377-0x10015bcaa160000, quorum=127.0.0.1:53474, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-11T16:21:59,265 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-11T16:21:59,279 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39377-0x10015bcaa160000, quorum=127.0.0.1:53474, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-11T16:21:59,279 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43823-0x10015bcaa160001, quorum=127.0.0.1:53474, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-11T16:21:59,279 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43823-0x10015bcaa160001, quorum=127.0.0.1:53474, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:21:59,279 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39377-0x10015bcaa160000, quorum=127.0.0.1:53474, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:21:59,282 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=7446a65196a1,39377,1733934116649, sessionid=0x10015bcaa160000, setting cluster-up flag (Was=false)
2024-12-11T16:21:59,315 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39377-0x10015bcaa160000, quorum=127.0.0.1:53474, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:21:59,316 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43823-0x10015bcaa160001, quorum=127.0.0.1:53474, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:21:59,357 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-11T16:21:59,359 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=7446a65196a1,39377,1733934116649
2024-12-11T16:21:59,390 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39377-0x10015bcaa160000, quorum=127.0.0.1:53474, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:21:59,390 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43823-0x10015bcaa160001, quorum=127.0.0.1:53474, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:21:59,431 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-11T16:21:59,434 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=7446a65196a1,39377,1733934116649
2024-12-11T16:21:59,442 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-12-11T16:21:59,465 INFO  [RS:0;7446a65196a1:43823 {}] regionserver.HRegionServer(746): ClusterId : 954ed961-52d2-4f35-a79a-e9949be00ee6
2024-12-11T16:21:59,468 DEBUG [RS:0;7446a65196a1:43823 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-11T16:21:59,489 DEBUG [RS:0;7446a65196a1:43823 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-11T16:21:59,490 DEBUG [RS:0;7446a65196a1:43823 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-11T16:21:59,520 DEBUG [RS:0;7446a65196a1:43823 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-11T16:21:59,521 DEBUG [RS:0;7446a65196a1:43823 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62b21a82, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7446a65196a1/172.17.0.2:0
2024-12-11T16:21:59,544 DEBUG [RS:0;7446a65196a1:43823 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;7446a65196a1:43823
2024-12-11T16:21:59,548 INFO  [RS:0;7446a65196a1:43823 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-12-11T16:21:59,548 INFO  [RS:0;7446a65196a1:43823 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-12-11T16:21:59,549 DEBUG [RS:0;7446a65196a1:43823 {}] regionserver.HRegionServer(832): About to register with Master.
2024-12-11T16:21:59,552 INFO  [RS:0;7446a65196a1:43823 {}] regionserver.HRegionServer(2659): reportForDuty to master=7446a65196a1,39377,1733934116649 with port=43823, startcode=1733934117800
2024-12-11T16:21:59,553 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-12-11T16:21:59,565 DEBUG [RS:0;7446a65196a1:43823 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-11T16:21:59,569 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-12-11T16:21:59,580 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-11T16:21:59,587 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 7446a65196a1,39377,1733934116649
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-11T16:21:59,612 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/7446a65196a1:0, corePoolSize=5, maxPoolSize=5
2024-12-11T16:21:59,612 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/7446a65196a1:0, corePoolSize=5, maxPoolSize=5
2024-12-11T16:21:59,612 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/7446a65196a1:0, corePoolSize=5, maxPoolSize=5
2024-12-11T16:21:59,613 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/7446a65196a1:0, corePoolSize=5, maxPoolSize=5
2024-12-11T16:21:59,613 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/7446a65196a1:0, corePoolSize=10, maxPoolSize=10
2024-12-11T16:21:59,613 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:21:59,613 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/7446a65196a1:0, corePoolSize=2, maxPoolSize=2
2024-12-11T16:21:59,613 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:21:59,634 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733934149634
2024-12-11T16:21:59,636 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-11T16:21:59,637 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-11T16:21:59,640 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-12-11T16:21:59,641 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-11T16:21:59,653 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-11T16:21:59,653 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-11T16:21:59,654 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-11T16:21:59,659 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-11T16:21:59,654 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:21:59,660 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-11T16:21:59,665 INFO  [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48177, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService
2024-12-11T16:21:59,663 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-11T16:21:59,674 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39377 {}] ipc.MetricsHBaseServer(152): Unknown exception type
org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet
	at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?]
	at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT]
2024-12-11T16:21:59,687 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-11T16:21:59,689 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-11T16:21:59,690 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-11T16:21:59,695 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-11T16:21:59,696 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-11T16:21:59,707 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/7446a65196a1:0:becomeActiveMaster-HFileCleaner.large.0-1733934119697,5,FailOnTimeoutGroup]
2024-12-11T16:21:59,711 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/7446a65196a1:0:becomeActiveMaster-HFileCleaner.small.0-1733934119707,5,FailOnTimeoutGroup]
2024-12-11T16:21:59,711 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-11T16:21:59,711 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-11T16:21:59,713 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-11T16:21:59,713 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-11T16:21:59,792 DEBUG [RS:0;7446a65196a1:43823 {}] regionserver.HRegionServer(2683): Master is not running yet
2024-12-11T16:21:59,792 WARN  [RS:0;7446a65196a1:43823 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying.
2024-12-11T16:21:59,880 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42931 is added to blk_1073741831_1007 (size=1321)
2024-12-11T16:21:59,880 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42705 is added to blk_1073741831_1007 (size=1321)
2024-12-11T16:21:59,894 INFO  [RS:0;7446a65196a1:43823 {}] regionserver.HRegionServer(2659): reportForDuty to master=7446a65196a1,39377,1733934116649 with port=43823, startcode=1733934117800
2024-12-11T16:21:59,897 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39377 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 7446a65196a1,43823,1733934117800
2024-12-11T16:21:59,900 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39377 {}] master.ServerManager(517): Registering regionserver=7446a65196a1,43823,1733934117800
2024-12-11T16:21:59,910 DEBUG [RS:0;7446a65196a1:43823 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c
2024-12-11T16:21:59,911 DEBUG [RS:0;7446a65196a1:43823 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:36921
2024-12-11T16:21:59,911 DEBUG [RS:0;7446a65196a1:43823 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-12-11T16:22:00,115 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39377-0x10015bcaa160000, quorum=127.0.0.1:53474, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-11T16:22:00,116 DEBUG [RS:0;7446a65196a1:43823 {}] zookeeper.ZKUtil(111): regionserver:43823-0x10015bcaa160001, quorum=127.0.0.1:53474, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/7446a65196a1,43823,1733934117800
2024-12-11T16:22:00,117 WARN  [RS:0;7446a65196a1:43823 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-11T16:22:00,117 INFO  [RS:0;7446a65196a1:43823 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-11T16:22:00,117 DEBUG [RS:0;7446a65196a1:43823 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/WALs/7446a65196a1,43823,1733934117800
2024-12-11T16:22:00,118 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [7446a65196a1,43823,1733934117800]
2024-12-11T16:22:00,147 INFO  [RS:0;7446a65196a1:43823 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-11T16:22:00,165 INFO  [RS:0;7446a65196a1:43823 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-11T16:22:00,171 INFO  [RS:0;7446a65196a1:43823 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-11T16:22:00,172 INFO  [RS:0;7446a65196a1:43823 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-11T16:22:00,176 INFO  [RS:0;7446a65196a1:43823 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-12-11T16:22:00,184 INFO  [RS:0;7446a65196a1:43823 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-12-11T16:22:00,186 INFO  [RS:0;7446a65196a1:43823 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-11T16:22:00,187 DEBUG [RS:0;7446a65196a1:43823 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:22:00,187 DEBUG [RS:0;7446a65196a1:43823 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:22:00,187 DEBUG [RS:0;7446a65196a1:43823 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:22:00,187 DEBUG [RS:0;7446a65196a1:43823 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:22:00,187 DEBUG [RS:0;7446a65196a1:43823 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:22:00,188 DEBUG [RS:0;7446a65196a1:43823 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/7446a65196a1:0, corePoolSize=2, maxPoolSize=2
2024-12-11T16:22:00,188 DEBUG [RS:0;7446a65196a1:43823 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:22:00,188 DEBUG [RS:0;7446a65196a1:43823 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:22:00,188 DEBUG [RS:0;7446a65196a1:43823 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:22:00,188 DEBUG [RS:0;7446a65196a1:43823 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:22:00,188 DEBUG [RS:0;7446a65196a1:43823 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:22:00,189 DEBUG [RS:0;7446a65196a1:43823 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:22:00,189 DEBUG [RS:0;7446a65196a1:43823 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/7446a65196a1:0, corePoolSize=3, maxPoolSize=3
2024-12-11T16:22:00,189 DEBUG [RS:0;7446a65196a1:43823 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/7446a65196a1:0, corePoolSize=3, maxPoolSize=3
2024-12-11T16:22:00,190 INFO  [RS:0;7446a65196a1:43823 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-11T16:22:00,191 INFO  [RS:0;7446a65196a1:43823 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-11T16:22:00,191 INFO  [RS:0;7446a65196a1:43823 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-11T16:22:00,191 INFO  [RS:0;7446a65196a1:43823 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-11T16:22:00,191 INFO  [RS:0;7446a65196a1:43823 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-11T16:22:00,191 INFO  [RS:0;7446a65196a1:43823 {}] hbase.ChoreService(168): Chore ScheduledChore name=7446a65196a1,43823,1733934117800-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-11T16:22:00,218 INFO  [RS:0;7446a65196a1:43823 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-11T16:22:00,220 INFO  [RS:0;7446a65196a1:43823 {}] hbase.ChoreService(168): Chore ScheduledChore name=7446a65196a1,43823,1733934117800-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-11T16:22:00,221 INFO  [RS:0;7446a65196a1:43823 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T16:22:00,221 INFO  [RS:0;7446a65196a1:43823 {}] regionserver.Replication(171): 7446a65196a1,43823,1733934117800 started
2024-12-11T16:22:00,246 INFO  [RS:0;7446a65196a1:43823 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T16:22:00,246 INFO  [RS:0;7446a65196a1:43823 {}] regionserver.HRegionServer(1482): Serving as 7446a65196a1,43823,1733934117800, RpcServer on 7446a65196a1/172.17.0.2:43823, sessionid=0x10015bcaa160001
2024-12-11T16:22:00,247 DEBUG [RS:0;7446a65196a1:43823 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-11T16:22:00,248 DEBUG [RS:0;7446a65196a1:43823 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 7446a65196a1,43823,1733934117800
2024-12-11T16:22:00,248 DEBUG [RS:0;7446a65196a1:43823 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7446a65196a1,43823,1733934117800'
2024-12-11T16:22:00,248 DEBUG [RS:0;7446a65196a1:43823 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-11T16:22:00,263 DEBUG [RS:0;7446a65196a1:43823 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-11T16:22:00,267 DEBUG [RS:0;7446a65196a1:43823 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-11T16:22:00,267 DEBUG [RS:0;7446a65196a1:43823 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-11T16:22:00,268 DEBUG [RS:0;7446a65196a1:43823 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 7446a65196a1,43823,1733934117800
2024-12-11T16:22:00,268 DEBUG [RS:0;7446a65196a1:43823 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7446a65196a1,43823,1733934117800'
2024-12-11T16:22:00,268 DEBUG [RS:0;7446a65196a1:43823 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-11T16:22:00,272 DEBUG [RS:0;7446a65196a1:43823 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-11T16:22:00,273 DEBUG [RS:0;7446a65196a1:43823 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-11T16:22:00,273 INFO  [RS:0;7446a65196a1:43823 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-11T16:22:00,273 INFO  [RS:0;7446a65196a1:43823 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-11T16:22:00,292 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-12-11T16:22:00,292 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c
2024-12-11T16:22:00,315 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42931 is added to blk_1073741832_1008 (size=32)
2024-12-11T16:22:00,318 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42705 is added to blk_1073741832_1008 (size=32)
2024-12-11T16:22:00,320 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T16:22:00,330 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-11T16:22:00,335 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-11T16:22:00,335 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:22:00,336 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T16:22:00,336 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-11T16:22:00,339 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-11T16:22:00,339 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:22:00,341 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T16:22:00,341 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-11T16:22:00,345 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-11T16:22:00,345 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:22:00,346 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T16:22:00,346 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-11T16:22:00,349 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-11T16:22:00,349 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:22:00,350 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T16:22:00,350 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-11T16:22:00,352 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/data/hbase/meta/1588230740
2024-12-11T16:22:00,353 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/data/hbase/meta/1588230740
2024-12-11T16:22:00,358 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-11T16:22:00,358 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-11T16:22:00,359 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-11T16:22:00,367 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-11T16:22:00,378 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-11T16:22:00,380 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=773245, jitterRate=-0.016768336296081543}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-11T16:22:00,383 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1733934120321Initializing all the Stores at 1733934120328 (+7 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733934120328Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733934120329 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733934120329Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733934120329Cleaning up temporary data from old regions at 1733934120358 (+29 ms)Region opened successfully at 1733934120383 (+25 ms)
2024-12-11T16:22:00,384 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-11T16:22:00,384 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-11T16:22:00,384 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-11T16:22:00,384 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-11T16:22:00,384 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-11T16:22:00,384 INFO  [RS:0;7446a65196a1:43823 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7446a65196a1%2C43823%2C1733934117800, suffix=, logDir=hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/WALs/7446a65196a1,43823,1733934117800, archiveDir=hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/oldWALs, maxLogs=32
2024-12-11T16:22:00,388 INFO  [RS:0;7446a65196a1:43823 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7446a65196a1%2C43823%2C1733934117800.1733934120388
2024-12-11T16:22:00,393 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-11T16:22:00,393 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733934120384Disabling compacts and flushes for region at 1733934120384Disabling writes for close at 1733934120384Writing region close event to WAL at 1733934120392 (+8 ms)Closed at 1733934120393 (+1 ms)
2024-12-11T16:22:00,402 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-11T16:22:00,402 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-12-11T16:22:00,412 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-11T16:22:00,437 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-11T16:22:00,447 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-11T16:22:00,471 INFO  [RS:0;7446a65196a1:43823 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/WALs/7446a65196a1,43823,1733934117800/7446a65196a1%2C43823%2C1733934117800.1733934120388
2024-12-11T16:22:00,495 DEBUG [RS:0;7446a65196a1:43823 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46071:46071),(127.0.0.1/127.0.0.1:41771:41771)]
2024-12-11T16:22:00,603 DEBUG [7446a65196a1:39377 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-12-11T16:22:00,620 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=7446a65196a1,43823,1733934117800
2024-12-11T16:22:00,628 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7446a65196a1,43823,1733934117800, state=OPENING
2024-12-11T16:22:00,656 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-11T16:22:00,670 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43823-0x10015bcaa160001, quorum=127.0.0.1:53474, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:22:00,670 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39377-0x10015bcaa160000, quorum=127.0.0.1:53474, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:22:00,671 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-11T16:22:00,671 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-11T16:22:00,673 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-11T16:22:00,676 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=7446a65196a1,43823,1733934117800}]
2024-12-11T16:22:00,857 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-11T16:22:00,864 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57837, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-11T16:22:00,880 INFO  [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-12-11T16:22:00,881 INFO  [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-11T16:22:00,889 INFO  [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7446a65196a1%2C43823%2C1733934117800.meta, suffix=.meta, logDir=hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/WALs/7446a65196a1,43823,1733934117800, archiveDir=hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/oldWALs, maxLogs=32
2024-12-11T16:22:00,892 INFO  [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 7446a65196a1%2C43823%2C1733934117800.meta.1733934120891.meta
2024-12-11T16:22:00,918 INFO  [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/WALs/7446a65196a1,43823,1733934117800/7446a65196a1%2C43823%2C1733934117800.meta.1733934120891.meta
2024-12-11T16:22:00,923 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41771:41771),(127.0.0.1/127.0.0.1:46071:46071)]
2024-12-11T16:22:00,934 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-11T16:22:00,937 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-11T16:22:00,941 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-11T16:22:00,952 INFO  [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-11T16:22:00,959 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-11T16:22:00,960 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T16:22:00,961 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-12-11T16:22:00,961 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-12-11T16:22:00,969 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-11T16:22:00,971 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-11T16:22:00,971 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:22:00,972 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T16:22:00,973 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-11T16:22:00,975 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-11T16:22:00,975 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:22:00,977 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T16:22:00,977 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-11T16:22:00,979 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-11T16:22:00,979 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:22:00,981 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T16:22:00,981 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-11T16:22:00,983 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-11T16:22:00,983 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:22:00,984 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T16:22:00,984 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-11T16:22:00,986 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/data/hbase/meta/1588230740
2024-12-11T16:22:00,990 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/data/hbase/meta/1588230740
2024-12-11T16:22:00,993 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-11T16:22:00,993 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-11T16:22:00,995 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-11T16:22:00,998 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-11T16:22:01,001 INFO  [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=747347, jitterRate=-0.0497003048658371}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-11T16:22:01,001 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-12-11T16:22:01,003 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1733934120962Writing region info on filesystem at 1733934120963 (+1 ms)Initializing all the Stores at 1733934120968 (+5 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733934120968Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733934120969 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733934120969Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733934120969Cleaning up temporary data from old regions at 1733934120994 (+25 ms)Running coprocessor post-open hooks at 1733934121002 (+8 ms)Region opened successfully at 1733934121003 (+1 ms)
2024-12-11T16:22:01,017 INFO  [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733934120844
2024-12-11T16:22:01,050 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-11T16:22:01,050 INFO  [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-12-11T16:22:01,054 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=7446a65196a1,43823,1733934117800
2024-12-11T16:22:01,060 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7446a65196a1,43823,1733934117800, state=OPEN
2024-12-11T16:22:01,119 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39377-0x10015bcaa160000, quorum=127.0.0.1:53474, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-11T16:22:01,119 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43823-0x10015bcaa160001, quorum=127.0.0.1:53474, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-11T16:22:01,119 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-11T16:22:01,119 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-11T16:22:01,119 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=7446a65196a1,43823,1733934117800
2024-12-11T16:22:01,129 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-12-11T16:22:01,129 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7446a65196a1,43823,1733934117800 in 445 msec
2024-12-11T16:22:01,147 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-12-11T16:22:01,148 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 720 msec
2024-12-11T16:22:01,152 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-11T16:22:01,152 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-12-11T16:22:01,177 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-11T16:22:01,179 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7446a65196a1,43823,1733934117800, seqNum=-1]
2024-12-11T16:22:01,210 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-11T16:22:01,213 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58001, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-11T16:22:01,284 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.7930 sec
2024-12-11T16:22:01,284 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733934121284, completionTime=-1
2024-12-11T16:22:01,287 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-12-11T16:22:01,287 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-12-11T16:22:01,328 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-12-11T16:22:01,329 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733934181328
2024-12-11T16:22:01,329 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733934241329
2024-12-11T16:22:01,329 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 41 msec
2024-12-11T16:22:01,334 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7446a65196a1,39377,1733934116649-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-11T16:22:01,335 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7446a65196a1,39377,1733934116649-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T16:22:01,336 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7446a65196a1,39377,1733934116649-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T16:22:01,338 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-7446a65196a1:39377, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T16:22:01,338 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-11T16:22:01,352 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-12-11T16:22:01,370 DEBUG [master/7446a65196a1:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-12-11T16:22:01,410 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.214sec
2024-12-11T16:22:01,412 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-11T16:22:01,414 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-11T16:22:01,416 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-11T16:22:01,419 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-11T16:22:01,419 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-11T16:22:01,420 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7446a65196a1,39377,1733934116649-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-11T16:22:01,421 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7446a65196a1,39377,1733934116649-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-11T16:22:01,438 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-12-11T16:22:01,439 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-11T16:22:01,440 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7446a65196a1,39377,1733934116649-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T16:22:01,496 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@145bbe65, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-11T16:22:01,511 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false
2024-12-11T16:22:01,511 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512
2024-12-11T16:22:01,516 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7446a65196a1,39377,-1 for getting cluster id
2024-12-11T16:22:01,522 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-12-11T16:22:01,538 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '954ed961-52d2-4f35-a79a-e9949be00ee6'
2024-12-11T16:22:01,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-12-11T16:22:01,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "954ed961-52d2-4f35-a79a-e9949be00ee6"

2024-12-11T16:22:01,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58524780, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-11T16:22:01,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7446a65196a1,39377,-1]
2024-12-11T16:22:01,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-12-11T16:22:01,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T16:22:01,567 INFO  [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42886, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-12-11T16:22:01,573 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@677cec9f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-11T16:22:01,574 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-11T16:22:01,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7446a65196a1,43823,1733934117800, seqNum=-1]
2024-12-11T16:22:01,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-11T16:22:01,590 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49562, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-11T16:22:01,620 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=7446a65196a1,39377,1733934116649
2024-12-11T16:22:01,620 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T16:22:01,633 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-12-11T16:22:01,639 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-12-11T16:22:01,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 7446a65196a1,39377,1733934116649
2024-12-11T16:22:01,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7c35b008
2024-12-11T16:22:01,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-12-11T16:22:01,662 INFO  [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42894, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-12-11T16:22:01,665 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39377 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-12-11T16:22:01,665 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39377 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-12-11T16:22:01,670 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39377 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testSlowSyncLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-11T16:22:01,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39377 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling
2024-12-11T16:22:01,694 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-12-11T16:22:01,697 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39377 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testSlowSyncLogRolling"
 procId is: 4
2024-12-11T16:22:01,698 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:22:01,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-11T16:22:01,711 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-12-11T16:22:01,770 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42931 is added to blk_1073741835_1011 (size=389)
2024-12-11T16:22:01,770 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42705 is added to blk_1073741835_1011 (size=389)
2024-12-11T16:22:01,778 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f8a0e31e333efe98bcc08c4d44983223, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1733934121664.f8a0e31e333efe98bcc08c4d44983223.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testSlowSyncLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c
2024-12-11T16:22:01,809 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42705 is added to blk_1073741836_1012 (size=72)
2024-12-11T16:22:01,810 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42931 is added to blk_1073741836_1012 (size=72)
2024-12-11T16:22:01,812 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testSlowSyncLogRolling,,1733934121664.f8a0e31e333efe98bcc08c4d44983223.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T16:22:01,812 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1722): Closing f8a0e31e333efe98bcc08c4d44983223, disabling compactions & flushes
2024-12-11T16:22:01,812 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testSlowSyncLogRolling,,1733934121664.f8a0e31e333efe98bcc08c4d44983223.
2024-12-11T16:22:01,812 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testSlowSyncLogRolling,,1733934121664.f8a0e31e333efe98bcc08c4d44983223.
2024-12-11T16:22:01,813 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1733934121664.f8a0e31e333efe98bcc08c4d44983223. after waiting 0 ms
2024-12-11T16:22:01,813 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1733934121664.f8a0e31e333efe98bcc08c4d44983223.
2024-12-11T16:22:01,813 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testSlowSyncLogRolling,,1733934121664.f8a0e31e333efe98bcc08c4d44983223.
2024-12-11T16:22:01,813 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for f8a0e31e333efe98bcc08c4d44983223:
Waiting for close lock at 1733934121812Disabling compacts and flushes for region at 1733934121812Disabling writes for close at 1733934121813 (+1 ms)Writing region close event to WAL at 1733934121813Closed at 1733934121813
2024-12-11T16:22:01,825 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ADD_TO_META
2024-12-11T16:22:01,831 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testSlowSyncLogRolling,,1733934121664.f8a0e31e333efe98bcc08c4d44983223.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1733934121825"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733934121825"}]},"ts":"1733934121825"}
2024-12-11T16:22:01,839 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-12-11T16:22:01,842 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-12-11T16:22:01,845 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733934121842"}]},"ts":"1733934121842"}
2024-12-11T16:22:01,853 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLING in hbase:meta
2024-12-11T16:22:01,856 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=f8a0e31e333efe98bcc08c4d44983223, ASSIGN}]
2024-12-11T16:22:01,859 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=f8a0e31e333efe98bcc08c4d44983223, ASSIGN
2024-12-11T16:22:01,862 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=f8a0e31e333efe98bcc08c4d44983223, ASSIGN; state=OFFLINE, location=7446a65196a1,43823,1733934117800; forceNewPlan=false, retain=false
2024-12-11T16:22:02,014 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=f8a0e31e333efe98bcc08c4d44983223, regionState=OPENING, regionLocation=7446a65196a1,43823,1733934117800
2024-12-11T16:22:02,020 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=f8a0e31e333efe98bcc08c4d44983223, ASSIGN because future has completed
2024-12-11T16:22:02,026 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure f8a0e31e333efe98bcc08c4d44983223, server=7446a65196a1,43823,1733934117800}]
2024-12-11T16:22:02,196 INFO  [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testSlowSyncLogRolling,,1733934121664.f8a0e31e333efe98bcc08c4d44983223.
2024-12-11T16:22:02,197 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => f8a0e31e333efe98bcc08c4d44983223, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1733934121664.f8a0e31e333efe98bcc08c4d44983223.', STARTKEY => '', ENDKEY => ''}
2024-12-11T16:22:02,197 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testSlowSyncLogRolling f8a0e31e333efe98bcc08c4d44983223
2024-12-11T16:22:02,198 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testSlowSyncLogRolling,,1733934121664.f8a0e31e333efe98bcc08c4d44983223.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T16:22:02,198 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for f8a0e31e333efe98bcc08c4d44983223
2024-12-11T16:22:02,198 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for f8a0e31e333efe98bcc08c4d44983223
2024-12-11T16:22:02,204 INFO  [StoreOpener-f8a0e31e333efe98bcc08c4d44983223-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region f8a0e31e333efe98bcc08c4d44983223 
2024-12-11T16:22:02,210 INFO  [StoreOpener-f8a0e31e333efe98bcc08c4d44983223-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f8a0e31e333efe98bcc08c4d44983223 columnFamilyName info
2024-12-11T16:22:02,210 DEBUG [StoreOpener-f8a0e31e333efe98bcc08c4d44983223-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:22:02,216 INFO  [StoreOpener-f8a0e31e333efe98bcc08c4d44983223-1 {}] regionserver.HStore(327): Store=f8a0e31e333efe98bcc08c4d44983223/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-11T16:22:02,216 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for f8a0e31e333efe98bcc08c4d44983223
2024-12-11T16:22:02,219 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/data/default/TestLogRolling-testSlowSyncLogRolling/f8a0e31e333efe98bcc08c4d44983223
2024-12-11T16:22:02,224 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/data/default/TestLogRolling-testSlowSyncLogRolling/f8a0e31e333efe98bcc08c4d44983223
2024-12-11T16:22:02,227 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for f8a0e31e333efe98bcc08c4d44983223
2024-12-11T16:22:02,228 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for f8a0e31e333efe98bcc08c4d44983223
2024-12-11T16:22:02,232 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for f8a0e31e333efe98bcc08c4d44983223
2024-12-11T16:22:02,241 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/data/default/TestLogRolling-testSlowSyncLogRolling/f8a0e31e333efe98bcc08c4d44983223/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-11T16:22:02,242 INFO  [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened f8a0e31e333efe98bcc08c4d44983223; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=706544, jitterRate=-0.10158367455005646}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-11T16:22:02,242 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f8a0e31e333efe98bcc08c4d44983223
2024-12-11T16:22:02,244 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for f8a0e31e333efe98bcc08c4d44983223:
Running coprocessor pre-open hook at 1733934122198Writing region info on filesystem at 1733934122198Initializing all the Stores at 1733934122203 (+5 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733934122204 (+1 ms)Cleaning up temporary data from old regions at 1733934122228 (+24 ms)Running coprocessor post-open hooks at 1733934122242 (+14 ms)Region opened successfully at 1733934122243 (+1 ms)
2024-12-11T16:22:02,259 INFO  [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testSlowSyncLogRolling,,1733934121664.f8a0e31e333efe98bcc08c4d44983223., pid=6, masterSystemTime=1733934122189
2024-12-11T16:22:02,269 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testSlowSyncLogRolling,,1733934121664.f8a0e31e333efe98bcc08c4d44983223.
2024-12-11T16:22:02,269 INFO  [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testSlowSyncLogRolling,,1733934121664.f8a0e31e333efe98bcc08c4d44983223.
2024-12-11T16:22:02,272 INFO  [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=f8a0e31e333efe98bcc08c4d44983223, regionState=OPEN, openSeqNum=2, regionLocation=7446a65196a1,43823,1733934117800
2024-12-11T16:22:02,284 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure f8a0e31e333efe98bcc08c4d44983223, server=7446a65196a1,43823,1733934117800 because future has completed
2024-12-11T16:22:02,291 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-12-11T16:22:02,293 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure f8a0e31e333efe98bcc08c4d44983223, server=7446a65196a1,43823,1733934117800 in 260 msec
2024-12-11T16:22:02,297 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-12-11T16:22:02,297 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=f8a0e31e333efe98bcc08c4d44983223, ASSIGN in 436 msec
2024-12-11T16:22:02,298 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-12-11T16:22:02,298 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733934122298"}]},"ts":"1733934122298"}
2024-12-11T16:22:02,303 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLED in hbase:meta
2024-12-11T16:22:02,305 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_POST_OPERATION
2024-12-11T16:22:02,309 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling in 631 msec
2024-12-11T16:22:06,470 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties
2024-12-11T16:22:06,526 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-12-11T16:22:06,528 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testSlowSyncLogRolling'
2024-12-11T16:22:07,192 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-12-11T16:22:07,193 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-12-11T16:22:07,195 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling
2024-12-11T16:22:07,195 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling Metrics about Tables on a single HBase RegionServer
2024-12-11T16:22:07,196 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-11T16:22:07,196 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-12-11T16:22:07,197 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-12-11T16:22:07,197 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer
2024-12-11T16:22:11,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-11T16:22:11,822 INFO  [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testSlowSyncLogRolling completed
2024-12-11T16:22:11,825 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testSlowSyncLogRolling,, stopping at row=TestLogRolling-testSlowSyncLogRolling ,, for max=2147483647 with caching=100
2024-12-11T16:22:11,833 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testSlowSyncLogRolling
2024-12-11T16:22:11,834 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testSlowSyncLogRolling,,1733934121664.f8a0e31e333efe98bcc08c4d44983223.
2024-12-11T16:22:11,835 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7446a65196a1%2C43823%2C1733934117800.1733934131835
2024-12-11T16:22:11,887 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:22:11,891 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:22:11,891 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:22:11,893 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:22:11,893 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:22:11,894 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/WALs/7446a65196a1,43823,1733934117800/7446a65196a1%2C43823%2C1733934117800.1733934120388 with entries=1, filesize=443 B; new WAL /user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/WALs/7446a65196a1,43823,1733934117800/7446a65196a1%2C43823%2C1733934117800.1733934131835
2024-12-11T16:22:11,906 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41771:41771),(127.0.0.1/127.0.0.1:46071:46071)]
2024-12-11T16:22:11,906 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/WALs/7446a65196a1,43823,1733934117800/7446a65196a1%2C43823%2C1733934117800.1733934120388 is not closed yet, will try archiving it next time
2024-12-11T16:22:11,907 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42931 is added to blk_1073741833_1009 (size=451)
2024-12-11T16:22:11,907 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42705 is added to blk_1073741833_1009 (size=451)
2024-12-11T16:22:11,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testSlowSyncLogRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testSlowSyncLogRolling,,1733934121664.f8a0e31e333efe98bcc08c4d44983223., hostname=7446a65196a1,43823,1733934117800, seqNum=2]
2024-12-11T16:22:12,309 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/WALs/7446a65196a1,43823,1733934117800/7446a65196a1%2C43823%2C1733934117800.1733934120388 to hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/oldWALs/7446a65196a1%2C43823%2C1733934117800.1733934120388
2024-12-11T16:22:23,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43823 {}] regionserver.HRegion(8855): Flush requested on f8a0e31e333efe98bcc08c4d44983223
2024-12-11T16:22:23,967 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f8a0e31e333efe98bcc08c4d44983223 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-11T16:22:24,085 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/data/default/TestLogRolling-testSlowSyncLogRolling/f8a0e31e333efe98bcc08c4d44983223/.tmp/info/83ee13b8ffa4461fb2b6f413299d2e69 is 1080, key is row0001/info:/1733934131925/Put/seqid=0
2024-12-11T16:22:24,129 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42931 is added to blk_1073741838_1014 (size=12509)
2024-12-11T16:22:24,133 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42705 is added to blk_1073741838_1014 (size=12509)
2024-12-11T16:22:24,139 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/data/default/TestLogRolling-testSlowSyncLogRolling/f8a0e31e333efe98bcc08c4d44983223/.tmp/info/83ee13b8ffa4461fb2b6f413299d2e69
2024-12-11T16:22:24,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/data/default/TestLogRolling-testSlowSyncLogRolling/f8a0e31e333efe98bcc08c4d44983223/.tmp/info/83ee13b8ffa4461fb2b6f413299d2e69 as hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/data/default/TestLogRolling-testSlowSyncLogRolling/f8a0e31e333efe98bcc08c4d44983223/info/83ee13b8ffa4461fb2b6f413299d2e69
2024-12-11T16:22:24,277 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/data/default/TestLogRolling-testSlowSyncLogRolling/f8a0e31e333efe98bcc08c4d44983223/info/83ee13b8ffa4461fb2b6f413299d2e69, entries=7, sequenceid=11, filesize=12.2 K
2024-12-11T16:22:24,286 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for f8a0e31e333efe98bcc08c4d44983223 in 318ms, sequenceid=11, compaction requested=false
2024-12-11T16:22:24,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f8a0e31e333efe98bcc08c4d44983223:

2024-12-11T16:22:25,615 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-11T16:22:31,975 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7446a65196a1%2C43823%2C1733934117800.1733934151975
2024-12-11T16:22:32,190 INFO  [Time-limited test {}] wal.AbstractFSWAL(1368): Slow sync cost: 212 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:42931,DS-0137187c-2bdf-42ef-886c-d633906df312,DISK], DatanodeInfoWithStorage[127.0.0.1:42705,DS-ae50a1ed-d15b-4f7e-b088-c3df73ccfe60,DISK]]
2024-12-11T16:22:32,190 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:22:32,190 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:22:32,191 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:22:32,191 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:22:32,195 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:22:32,195 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/WALs/7446a65196a1,43823,1733934117800/7446a65196a1%2C43823%2C1733934117800.1733934131835 with entries=12, filesize=12.10 KB; new WAL /user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/WALs/7446a65196a1,43823,1733934117800/7446a65196a1%2C43823%2C1733934117800.1733934151975
2024-12-11T16:22:32,198 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42705 is added to blk_1073741837_1013 (size=12399)
2024-12-11T16:22:32,198 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42931 is added to blk_1073741837_1013 (size=12399)
2024-12-11T16:22:32,201 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41771:41771),(127.0.0.1/127.0.0.1:46071:46071)]
2024-12-11T16:22:32,201 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/WALs/7446a65196a1,43823,1733934117800/7446a65196a1%2C43823%2C1733934117800.1733934131835 is not closed yet, will try archiving it next time
2024-12-11T16:22:32,404 INFO  [FSHLog-0-hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c-prefix:7446a65196a1,43823,1733934117800 {}] wal.AbstractFSWAL(1368): Slow sync cost: 200 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:42931,DS-0137187c-2bdf-42ef-886c-d633906df312,DISK], DatanodeInfoWithStorage[127.0.0.1:42705,DS-ae50a1ed-d15b-4f7e-b088-c3df73ccfe60,DISK]]
2024-12-11T16:22:34,608 INFO  [FSHLog-0-hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c-prefix:7446a65196a1,43823,1733934117800 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:42931,DS-0137187c-2bdf-42ef-886c-d633906df312,DISK], DatanodeInfoWithStorage[127.0.0.1:42705,DS-ae50a1ed-d15b-4f7e-b088-c3df73ccfe60,DISK]]
2024-12-11T16:22:36,813 INFO  [FSHLog-0-hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c-prefix:7446a65196a1,43823,1733934117800 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:42931,DS-0137187c-2bdf-42ef-886c-d633906df312,DISK], DatanodeInfoWithStorage[127.0.0.1:42705,DS-ae50a1ed-d15b-4f7e-b088-c3df73ccfe60,DISK]]
2024-12-11T16:22:39,017 INFO  [FSHLog-0-hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c-prefix:7446a65196a1,43823,1733934117800 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:42931,DS-0137187c-2bdf-42ef-886c-d633906df312,DISK], DatanodeInfoWithStorage[127.0.0.1:42705,DS-ae50a1ed-d15b-4f7e-b088-c3df73ccfe60,DISK]]
2024-12-11T16:22:39,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43823 {}] regionserver.HRegion(8855): Flush requested on f8a0e31e333efe98bcc08c4d44983223
2024-12-11T16:22:39,018 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f8a0e31e333efe98bcc08c4d44983223 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-11T16:22:39,221 INFO  [FSHLog-0-hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c-prefix:7446a65196a1,43823,1733934117800 {}] wal.AbstractFSWAL(1368): Slow sync cost: 202 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:42931,DS-0137187c-2bdf-42ef-886c-d633906df312,DISK], DatanodeInfoWithStorage[127.0.0.1:42705,DS-ae50a1ed-d15b-4f7e-b088-c3df73ccfe60,DISK]]
2024-12-11T16:22:39,228 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/data/default/TestLogRolling-testSlowSyncLogRolling/f8a0e31e333efe98bcc08c4d44983223/.tmp/info/e7802e6da06a435d925894016d96ec95 is 1080, key is row0008/info:/1733934145965/Put/seqid=0
2024-12-11T16:22:39,238 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42931 is added to blk_1073741840_1016 (size=12509)
2024-12-11T16:22:39,238 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42705 is added to blk_1073741840_1016 (size=12509)
2024-12-11T16:22:39,640 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/data/default/TestLogRolling-testSlowSyncLogRolling/f8a0e31e333efe98bcc08c4d44983223/.tmp/info/e7802e6da06a435d925894016d96ec95
2024-12-11T16:22:39,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/data/default/TestLogRolling-testSlowSyncLogRolling/f8a0e31e333efe98bcc08c4d44983223/.tmp/info/e7802e6da06a435d925894016d96ec95 as hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/data/default/TestLogRolling-testSlowSyncLogRolling/f8a0e31e333efe98bcc08c4d44983223/info/e7802e6da06a435d925894016d96ec95
2024-12-11T16:22:39,670 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/data/default/TestLogRolling-testSlowSyncLogRolling/f8a0e31e333efe98bcc08c4d44983223/info/e7802e6da06a435d925894016d96ec95, entries=7, sequenceid=21, filesize=12.2 K
2024-12-11T16:22:39,872 INFO  [FSHLog-0-hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c-prefix:7446a65196a1,43823,1733934117800 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:42931,DS-0137187c-2bdf-42ef-886c-d633906df312,DISK], DatanodeInfoWithStorage[127.0.0.1:42705,DS-ae50a1ed-d15b-4f7e-b088-c3df73ccfe60,DISK]]
2024-12-11T16:22:39,873 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for f8a0e31e333efe98bcc08c4d44983223 in 855ms, sequenceid=21, compaction requested=false
2024-12-11T16:22:39,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f8a0e31e333efe98bcc08c4d44983223:

2024-12-11T16:22:39,873 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=24.4 K, sizeToCheck=16.0 K
2024-12-11T16:22:39,873 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-11T16:22:39,874 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/data/default/TestLogRolling-testSlowSyncLogRolling/f8a0e31e333efe98bcc08c4d44983223/info/83ee13b8ffa4461fb2b6f413299d2e69 because midkey is the same as first or last row
2024-12-11T16:22:41,222 INFO  [FSHLog-0-hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c-prefix:7446a65196a1,43823,1733934117800 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:42931,DS-0137187c-2bdf-42ef-886c-d633906df312,DISK], DatanodeInfoWithStorage[127.0.0.1:42705,DS-ae50a1ed-d15b-4f7e-b088-c3df73ccfe60,DISK]]
2024-12-11T16:22:41,491 INFO  [master/7446a65196a1:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-12-11T16:22:41,491 INFO  [master/7446a65196a1:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-12-11T16:22:43,426 INFO  [FSHLog-0-hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c-prefix:7446a65196a1,43823,1733934117800 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:42931,DS-0137187c-2bdf-42ef-886c-d633906df312,DISK], DatanodeInfoWithStorage[127.0.0.1:42705,DS-ae50a1ed-d15b-4f7e-b088-c3df73ccfe60,DISK]]
2024-12-11T16:22:43,429 WARN  [FSHLog-0-hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c-prefix:7446a65196a1,43823,1733934117800 {}] wal.AbstractFSWAL(2201): Requesting log roll because we exceeded slow sync threshold; count=8, threshold=5, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:42931,DS-0137187c-2bdf-42ef-886c-d633906df312,DISK], DatanodeInfoWithStorage[127.0.0.1:42705,DS-ae50a1ed-d15b-4f7e-b088-c3df73ccfe60,DISK]]
2024-12-11T16:22:43,430 DEBUG [regionserver/7446a65196a1:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 7446a65196a1%2C43823%2C1733934117800:(num 1733934151975) roll requested
2024-12-11T16:22:43,431 INFO  [regionserver/7446a65196a1:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7446a65196a1%2C43823%2C1733934117800.1733934163431
2024-12-11T16:22:43,643 INFO  [regionserver/7446a65196a1:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 209 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:42931,DS-0137187c-2bdf-42ef-886c-d633906df312,DISK], DatanodeInfoWithStorage[127.0.0.1:42705,DS-ae50a1ed-d15b-4f7e-b088-c3df73ccfe60,DISK]]
2024-12-11T16:22:43,643 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:22:43,643 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:22:43,644 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:22:43,644 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:22:43,644 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:22:43,644 INFO  [regionserver/7446a65196a1:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/WALs/7446a65196a1,43823,1733934117800/7446a65196a1%2C43823%2C1733934117800.1733934151975 with entries=8, filesize=7.55 KB; new WAL /user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/WALs/7446a65196a1,43823,1733934117800/7446a65196a1%2C43823%2C1733934117800.1733934163431
2024-12-11T16:22:43,646 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42705 is added to blk_1073741839_1015 (size=7739)
2024-12-11T16:22:43,647 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42931 is added to blk_1073741839_1015 (size=7739)
2024-12-11T16:22:43,647 DEBUG [regionserver/7446a65196a1:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46071:46071),(127.0.0.1/127.0.0.1:41771:41771)]
2024-12-11T16:22:43,647 DEBUG [regionserver/7446a65196a1:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/WALs/7446a65196a1,43823,1733934117800/7446a65196a1%2C43823%2C1733934117800.1733934151975 is not closed yet, will try archiving it next time
2024-12-11T16:22:43,647 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/WALs/7446a65196a1,43823,1733934117800/7446a65196a1%2C43823%2C1733934117800.1733934131835 to hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/oldWALs/7446a65196a1%2C43823%2C1733934117800.1733934131835
2024-12-11T16:22:45,633 INFO  [FSHLog-0-hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c-prefix:7446a65196a1,43823,1733934117800 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:42705,DS-ae50a1ed-d15b-4f7e-b088-c3df73ccfe60,DISK], DatanodeInfoWithStorage[127.0.0.1:42931,DS-0137187c-2bdf-42ef-886c-d633906df312,DISK]]
2024-12-11T16:22:47,198 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region f8a0e31e333efe98bcc08c4d44983223, had cached 0 bytes from a total of 25018
2024-12-11T16:22:47,838 INFO  [FSHLog-0-hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c-prefix:7446a65196a1,43823,1733934117800 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:42705,DS-ae50a1ed-d15b-4f7e-b088-c3df73ccfe60,DISK], DatanodeInfoWithStorage[127.0.0.1:42931,DS-0137187c-2bdf-42ef-886c-d633906df312,DISK]]
2024-12-11T16:22:50,043 INFO  [FSHLog-0-hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c-prefix:7446a65196a1,43823,1733934117800 {}] wal.AbstractFSWAL(1368): Slow sync cost: 200 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:42705,DS-ae50a1ed-d15b-4f7e-b088-c3df73ccfe60,DISK], DatanodeInfoWithStorage[127.0.0.1:42931,DS-0137187c-2bdf-42ef-886c-d633906df312,DISK]]
2024-12-11T16:22:52,247 INFO  [FSHLog-0-hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c-prefix:7446a65196a1,43823,1733934117800 {}] wal.AbstractFSWAL(1368): Slow sync cost: 200 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:42705,DS-ae50a1ed-d15b-4f7e-b088-c3df73ccfe60,DISK], DatanodeInfoWithStorage[127.0.0.1:42931,DS-0137187c-2bdf-42ef-886c-d633906df312,DISK]]
2024-12-11T16:22:54,250 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2024-12-11T16:22:54,250 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7446a65196a1%2C43823%2C1733934117800.1733934174250
2024-12-11T16:22:55,615 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-11T16:22:59,293 INFO  [Time-limited test {}] wal.AbstractFSWAL(1368): Slow sync cost: 5034 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:42705,DS-ae50a1ed-d15b-4f7e-b088-c3df73ccfe60,DISK], DatanodeInfoWithStorage[127.0.0.1:42931,DS-0137187c-2bdf-42ef-886c-d633906df312,DISK]]
2024-12-11T16:22:59,296 WARN  [Time-limited test {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5034 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:42705,DS-ae50a1ed-d15b-4f7e-b088-c3df73ccfe60,DISK], DatanodeInfoWithStorage[127.0.0.1:42931,DS-0137187c-2bdf-42ef-886c-d633906df312,DISK]]
2024-12-11T16:22:59,296 DEBUG [regionserver/7446a65196a1:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 7446a65196a1%2C43823%2C1733934117800:(num 1733934174250) roll requested
2024-12-11T16:22:59,296 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:22:59,297 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:22:59,297 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:22:59,299 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:22:59,302 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:22:59,304 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/WALs/7446a65196a1,43823,1733934117800/7446a65196a1%2C43823%2C1733934117800.1733934163431 with entries=4, filesize=4.63 KB; new WAL /user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/WALs/7446a65196a1,43823,1733934117800/7446a65196a1%2C43823%2C1733934117800.1733934174250
2024-12-11T16:22:59,316 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46071:46071),(127.0.0.1/127.0.0.1:41771:41771)]
2024-12-11T16:22:59,317 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/WALs/7446a65196a1,43823,1733934117800/7446a65196a1%2C43823%2C1733934117800.1733934163431 is not closed yet, will try archiving it next time
2024-12-11T16:22:59,317 INFO  [regionserver/7446a65196a1:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7446a65196a1%2C43823%2C1733934117800.1733934179317
2024-12-11T16:22:59,324 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42931 is added to blk_1073741841_1017 (size=4753)
2024-12-11T16:22:59,325 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42705 is added to blk_1073741841_1017 (size=4753)
2024-12-11T16:23:04,325 INFO  [FSHLog-0-hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c-prefix:7446a65196a1,43823,1733934117800 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5001 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:42705,DS-ae50a1ed-d15b-4f7e-b088-c3df73ccfe60,DISK], DatanodeInfoWithStorage[127.0.0.1:42931,DS-0137187c-2bdf-42ef-886c-d633906df312,DISK]]
2024-12-11T16:23:04,325 WARN  [FSHLog-0-hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c-prefix:7446a65196a1,43823,1733934117800 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5001 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:42705,DS-ae50a1ed-d15b-4f7e-b088-c3df73ccfe60,DISK], DatanodeInfoWithStorage[127.0.0.1:42931,DS-0137187c-2bdf-42ef-886c-d633906df312,DISK]]
2024-12-11T16:23:04,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43823 {}] regionserver.HRegion(8855): Flush requested on f8a0e31e333efe98bcc08c4d44983223
2024-12-11T16:23:04,325 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f8a0e31e333efe98bcc08c4d44983223 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-11T16:23:04,352 INFO  [regionserver/7446a65196a1:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 5017 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:42705,DS-ae50a1ed-d15b-4f7e-b088-c3df73ccfe60,DISK], DatanodeInfoWithStorage[127.0.0.1:42931,DS-0137187c-2bdf-42ef-886c-d633906df312,DISK]]
2024-12-11T16:23:04,352 WARN  [regionserver/7446a65196a1:0.logRoller {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5017 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:42705,DS-ae50a1ed-d15b-4f7e-b088-c3df73ccfe60,DISK], DatanodeInfoWithStorage[127.0.0.1:42931,DS-0137187c-2bdf-42ef-886c-d633906df312,DISK]]
2024-12-11T16:23:06,326 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2024-12-11T16:23:09,328 INFO  [FSHLog-0-hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c-prefix:7446a65196a1,43823,1733934117800 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5001 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:42705,DS-ae50a1ed-d15b-4f7e-b088-c3df73ccfe60,DISK], DatanodeInfoWithStorage[127.0.0.1:42931,DS-0137187c-2bdf-42ef-886c-d633906df312,DISK]]
2024-12-11T16:23:09,328 WARN  [FSHLog-0-hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c-prefix:7446a65196a1,43823,1733934117800 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5001 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:42705,DS-ae50a1ed-d15b-4f7e-b088-c3df73ccfe60,DISK], DatanodeInfoWithStorage[127.0.0.1:42931,DS-0137187c-2bdf-42ef-886c-d633906df312,DISK]]
2024-12-11T16:23:09,328 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:23:09,328 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:23:09,328 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:23:09,329 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:23:09,329 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:23:09,329 INFO  [regionserver/7446a65196a1:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/WALs/7446a65196a1,43823,1733934117800/7446a65196a1%2C43823%2C1733934117800.1733934174250 with entries=2, filesize=1.52 KB; new WAL /user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/WALs/7446a65196a1,43823,1733934117800/7446a65196a1%2C43823%2C1733934117800.1733934179317
2024-12-11T16:23:09,332 DEBUG [regionserver/7446a65196a1:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46071:46071),(127.0.0.1/127.0.0.1:41771:41771)]
2024-12-11T16:23:09,332 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42931 is added to blk_1073741842_1018 (size=1569)
2024-12-11T16:23:09,333 DEBUG [regionserver/7446a65196a1:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/WALs/7446a65196a1,43823,1733934117800/7446a65196a1%2C43823%2C1733934117800.1733934174250 is not closed yet, will try archiving it next time
2024-12-11T16:23:09,333 DEBUG [regionserver/7446a65196a1:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 7446a65196a1%2C43823%2C1733934117800:(num 1733934179317) roll requested
2024-12-11T16:23:09,333 INFO  [regionserver/7446a65196a1:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7446a65196a1%2C43823%2C1733934117800.1733934189333
2024-12-11T16:23:09,341 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42705 is added to blk_1073741842_1018 (size=1569)
2024-12-11T16:23:09,343 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/data/default/TestLogRolling-testSlowSyncLogRolling/f8a0e31e333efe98bcc08c4d44983223/.tmp/info/2aea13df3fba4ce1837ecd226a44bfb7 is 1080, key is row0015/info:/1733934161020/Put/seqid=0
2024-12-11T16:23:09,357 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42705 is added to blk_1073741844_1020 (size=12509)
2024-12-11T16:23:09,358 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42931 is added to blk_1073741844_1020 (size=12509)
2024-12-11T16:23:09,362 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=31 (bloomFilter=true), to=hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/data/default/TestLogRolling-testSlowSyncLogRolling/f8a0e31e333efe98bcc08c4d44983223/.tmp/info/2aea13df3fba4ce1837ecd226a44bfb7
2024-12-11T16:23:09,383 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/data/default/TestLogRolling-testSlowSyncLogRolling/f8a0e31e333efe98bcc08c4d44983223/.tmp/info/2aea13df3fba4ce1837ecd226a44bfb7 as hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/data/default/TestLogRolling-testSlowSyncLogRolling/f8a0e31e333efe98bcc08c4d44983223/info/2aea13df3fba4ce1837ecd226a44bfb7
2024-12-11T16:23:09,408 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/data/default/TestLogRolling-testSlowSyncLogRolling/f8a0e31e333efe98bcc08c4d44983223/info/2aea13df3fba4ce1837ecd226a44bfb7, entries=7, sequenceid=31, filesize=12.2 K
2024-12-11T16:23:14,349 INFO  [regionserver/7446a65196a1:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 5006 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:42705,DS-ae50a1ed-d15b-4f7e-b088-c3df73ccfe60,DISK], DatanodeInfoWithStorage[127.0.0.1:42931,DS-0137187c-2bdf-42ef-886c-d633906df312,DISK]]
2024-12-11T16:23:14,349 WARN  [regionserver/7446a65196a1:0.logRoller {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5006 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:42705,DS-ae50a1ed-d15b-4f7e-b088-c3df73ccfe60,DISK], DatanodeInfoWithStorage[127.0.0.1:42931,DS-0137187c-2bdf-42ef-886c-d633906df312,DISK]]
2024-12-11T16:23:14,415 INFO  [FSHLog-0-hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c-prefix:7446a65196a1,43823,1733934117800 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5003 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:42705,DS-ae50a1ed-d15b-4f7e-b088-c3df73ccfe60,DISK], DatanodeInfoWithStorage[127.0.0.1:42931,DS-0137187c-2bdf-42ef-886c-d633906df312,DISK]]
2024-12-11T16:23:14,415 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for f8a0e31e333efe98bcc08c4d44983223 in 10090ms, sequenceid=31, compaction requested=true
2024-12-11T16:23:14,415 WARN  [FSHLog-0-hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c-prefix:7446a65196a1,43823,1733934117800 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5003 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:42705,DS-ae50a1ed-d15b-4f7e-b088-c3df73ccfe60,DISK], DatanodeInfoWithStorage[127.0.0.1:42931,DS-0137187c-2bdf-42ef-886c-d633906df312,DISK]]
2024-12-11T16:23:14,415 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f8a0e31e333efe98bcc08c4d44983223:

2024-12-11T16:23:14,415 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:23:14,415 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:23:14,415 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=36.6 K, sizeToCheck=16.0 K
2024-12-11T16:23:14,415 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-11T16:23:14,415 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:23:14,415 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:23:14,415 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/data/default/TestLogRolling-testSlowSyncLogRolling/f8a0e31e333efe98bcc08c4d44983223/info/83ee13b8ffa4461fb2b6f413299d2e69 because midkey is the same as first or last row
2024-12-11T16:23:14,416 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:23:14,416 INFO  [regionserver/7446a65196a1:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/WALs/7446a65196a1,43823,1733934117800/7446a65196a1%2C43823%2C1733934117800.1733934179317 with entries=1, filesize=430 B; new WAL /user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/WALs/7446a65196a1,43823,1733934117800/7446a65196a1%2C43823%2C1733934117800.1733934189333
2024-12-11T16:23:14,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8a0e31e333efe98bcc08c4d44983223:info, priority=-2147483648, current under compaction store size is 1
2024-12-11T16:23:14,418 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42931 is added to blk_1073741843_1019 (size=438)
2024-12-11T16:23:14,418 DEBUG [regionserver/7446a65196a1:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41771:41771),(127.0.0.1/127.0.0.1:46071:46071)]
2024-12-11T16:23:14,419 DEBUG [regionserver/7446a65196a1:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/WALs/7446a65196a1,43823,1733934117800/7446a65196a1%2C43823%2C1733934117800.1733934179317 is not closed yet, will try archiving it next time
2024-12-11T16:23:14,419 DEBUG [regionserver/7446a65196a1:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 7446a65196a1%2C43823%2C1733934117800:(num 1733934189333) roll requested
2024-12-11T16:23:14,419 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42705 is added to blk_1073741843_1019 (size=438)
2024-12-11T16:23:14,419 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7446a65196a1%2C43823%2C1733934117800.1733934194419
2024-12-11T16:23:14,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-11T16:23:14,422 DEBUG [RS:0;7446a65196a1:43823-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-11T16:23:14,423 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/WALs/7446a65196a1,43823,1733934117800/7446a65196a1%2C43823%2C1733934117800.1733934151975 to hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/oldWALs/7446a65196a1%2C43823%2C1733934117800.1733934151975
2024-12-11T16:23:14,425 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/WALs/7446a65196a1,43823,1733934117800/7446a65196a1%2C43823%2C1733934117800.1733934163431 to hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/oldWALs/7446a65196a1%2C43823%2C1733934117800.1733934163431
2024-12-11T16:23:14,427 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/WALs/7446a65196a1,43823,1733934117800/7446a65196a1%2C43823%2C1733934117800.1733934174250 to hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/oldWALs/7446a65196a1%2C43823%2C1733934117800.1733934174250
2024-12-11T16:23:14,430 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/WALs/7446a65196a1,43823,1733934117800/7446a65196a1%2C43823%2C1733934117800.1733934179317 to hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/oldWALs/7446a65196a1%2C43823%2C1733934117800.1733934179317
2024-12-11T16:23:14,436 DEBUG [RS:0;7446a65196a1:43823-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 37527 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-11T16:23:14,437 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:23:14,437 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:23:14,437 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:23:14,438 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:23:14,438 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:23:14,438 DEBUG [RS:0;7446a65196a1:43823-shortCompactions-0 {}] regionserver.HStore(1541): f8a0e31e333efe98bcc08c4d44983223/info is initiating minor compaction (all files)
2024-12-11T16:23:14,438 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/WALs/7446a65196a1,43823,1733934117800/7446a65196a1%2C43823%2C1733934117800.1733934189333 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/WALs/7446a65196a1,43823,1733934117800/7446a65196a1%2C43823%2C1733934117800.1733934194419
2024-12-11T16:23:14,439 INFO  [RS:0;7446a65196a1:43823-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8a0e31e333efe98bcc08c4d44983223/info in TestLogRolling-testSlowSyncLogRolling,,1733934121664.f8a0e31e333efe98bcc08c4d44983223.
2024-12-11T16:23:14,440 INFO  [RS:0;7446a65196a1:43823-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/data/default/TestLogRolling-testSlowSyncLogRolling/f8a0e31e333efe98bcc08c4d44983223/info/83ee13b8ffa4461fb2b6f413299d2e69, hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/data/default/TestLogRolling-testSlowSyncLogRolling/f8a0e31e333efe98bcc08c4d44983223/info/e7802e6da06a435d925894016d96ec95, hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/data/default/TestLogRolling-testSlowSyncLogRolling/f8a0e31e333efe98bcc08c4d44983223/info/2aea13df3fba4ce1837ecd226a44bfb7] into tmpdir=hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/data/default/TestLogRolling-testSlowSyncLogRolling/f8a0e31e333efe98bcc08c4d44983223/.tmp, totalSize=36.6 K
2024-12-11T16:23:14,442 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42705 is added to blk_1073741845_1021 (size=93)
2024-12-11T16:23:14,443 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42931 is added to blk_1073741845_1021 (size=93)
2024-12-11T16:23:14,447 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/WALs/7446a65196a1,43823,1733934117800/7446a65196a1%2C43823%2C1733934117800.1733934189333 to hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/oldWALs/7446a65196a1%2C43823%2C1733934117800.1733934189333
2024-12-11T16:23:14,447 DEBUG [RS:0;7446a65196a1:43823-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83ee13b8ffa4461fb2b6f413299d2e69, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733934131925
2024-12-11T16:23:14,453 DEBUG [RS:0;7446a65196a1:43823-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7802e6da06a435d925894016d96ec95, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=21, earliestPutTs=1733934145965
2024-12-11T16:23:14,456 DEBUG [RS:0;7446a65196a1:43823-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2aea13df3fba4ce1837ecd226a44bfb7, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=31, earliestPutTs=1733934161020
2024-12-11T16:23:14,463 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41771:41771),(127.0.0.1/127.0.0.1:46071:46071)]
2024-12-11T16:23:14,465 INFO  [regionserver/7446a65196a1:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7446a65196a1%2C43823%2C1733934117800.1733934194465
2024-12-11T16:23:14,480 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:23:14,481 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:23:14,481 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:23:14,481 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:23:14,481 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:23:14,481 INFO  [regionserver/7446a65196a1:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/WALs/7446a65196a1,43823,1733934117800/7446a65196a1%2C43823%2C1733934117800.1733934194419 with entries=1, filesize=1.22 KB; new WAL /user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/WALs/7446a65196a1,43823,1733934117800/7446a65196a1%2C43823%2C1733934117800.1733934194465
2024-12-11T16:23:14,483 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42705 is added to blk_1073741846_1022 (size=1258)
2024-12-11T16:23:14,484 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42931 is added to blk_1073741846_1022 (size=1258)
2024-12-11T16:23:14,484 DEBUG [regionserver/7446a65196a1:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46071:46071),(127.0.0.1/127.0.0.1:41771:41771)]
2024-12-11T16:23:14,484 DEBUG [regionserver/7446a65196a1:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/WALs/7446a65196a1,43823,1733934117800/7446a65196a1%2C43823%2C1733934117800.1733934194419 is not closed yet, will try archiving it next time
2024-12-11T16:23:14,497 INFO  [RS:0;7446a65196a1:43823-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8a0e31e333efe98bcc08c4d44983223#info#compaction#3 average throughput is 10.77 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-11T16:23:14,498 DEBUG [RS:0;7446a65196a1:43823-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/data/default/TestLogRolling-testSlowSyncLogRolling/f8a0e31e333efe98bcc08c4d44983223/.tmp/info/59e5482cd032486499ded6e13106beea is 1080, key is row0001/info:/1733934131925/Put/seqid=0
2024-12-11T16:23:14,505 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42931 is added to blk_1073741848_1024 (size=27710)
2024-12-11T16:23:14,505 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42705 is added to blk_1073741848_1024 (size=27710)
2024-12-11T16:23:14,518 DEBUG [RS:0;7446a65196a1:43823-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/data/default/TestLogRolling-testSlowSyncLogRolling/f8a0e31e333efe98bcc08c4d44983223/.tmp/info/59e5482cd032486499ded6e13106beea as hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/data/default/TestLogRolling-testSlowSyncLogRolling/f8a0e31e333efe98bcc08c4d44983223/info/59e5482cd032486499ded6e13106beea
2024-12-11T16:23:14,539 INFO  [RS:0;7446a65196a1:43823-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8a0e31e333efe98bcc08c4d44983223/info of f8a0e31e333efe98bcc08c4d44983223 into 59e5482cd032486499ded6e13106beea(size=27.1 K), total size for store is 27.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-11T16:23:14,539 DEBUG [RS:0;7446a65196a1:43823-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8a0e31e333efe98bcc08c4d44983223:

2024-12-11T16:23:14,541 INFO  [RS:0;7446a65196a1:43823-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testSlowSyncLogRolling,,1733934121664.f8a0e31e333efe98bcc08c4d44983223., storeName=f8a0e31e333efe98bcc08c4d44983223/info, priority=13, startTime=1733934194417; duration=0sec
2024-12-11T16:23:14,541 DEBUG [RS:0;7446a65196a1:43823-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K
2024-12-11T16:23:14,542 DEBUG [RS:0;7446a65196a1:43823-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-11T16:23:14,542 DEBUG [RS:0;7446a65196a1:43823-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/data/default/TestLogRolling-testSlowSyncLogRolling/f8a0e31e333efe98bcc08c4d44983223/info/59e5482cd032486499ded6e13106beea because midkey is the same as first or last row
2024-12-11T16:23:14,542 DEBUG [RS:0;7446a65196a1:43823-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K
2024-12-11T16:23:14,542 DEBUG [RS:0;7446a65196a1:43823-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-11T16:23:14,542 DEBUG [RS:0;7446a65196a1:43823-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/data/default/TestLogRolling-testSlowSyncLogRolling/f8a0e31e333efe98bcc08c4d44983223/info/59e5482cd032486499ded6e13106beea because midkey is the same as first or last row
2024-12-11T16:23:14,542 DEBUG [RS:0;7446a65196a1:43823-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K
2024-12-11T16:23:14,542 DEBUG [RS:0;7446a65196a1:43823-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-11T16:23:14,543 DEBUG [RS:0;7446a65196a1:43823-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/data/default/TestLogRolling-testSlowSyncLogRolling/f8a0e31e333efe98bcc08c4d44983223/info/59e5482cd032486499ded6e13106beea because midkey is the same as first or last row
2024-12-11T16:23:14,543 DEBUG [RS:0;7446a65196a1:43823-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-11T16:23:14,543 DEBUG [RS:0;7446a65196a1:43823-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8a0e31e333efe98bcc08c4d44983223:info
2024-12-11T16:23:25,616 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-11T16:23:26,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43823 {}] regionserver.HRegion(8855): Flush requested on f8a0e31e333efe98bcc08c4d44983223
2024-12-11T16:23:26,493 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f8a0e31e333efe98bcc08c4d44983223 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-11T16:23:26,502 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/data/default/TestLogRolling-testSlowSyncLogRolling/f8a0e31e333efe98bcc08c4d44983223/.tmp/info/ecb8d22b97c74f96bd0475ba204a61ea is 1080, key is row0022/info:/1733934194465/Put/seqid=0
2024-12-11T16:23:26,518 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42931 is added to blk_1073741849_1025 (size=12509)
2024-12-11T16:23:26,519 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42705 is added to blk_1073741849_1025 (size=12509)
2024-12-11T16:23:26,519 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/data/default/TestLogRolling-testSlowSyncLogRolling/f8a0e31e333efe98bcc08c4d44983223/.tmp/info/ecb8d22b97c74f96bd0475ba204a61ea
2024-12-11T16:23:26,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/data/default/TestLogRolling-testSlowSyncLogRolling/f8a0e31e333efe98bcc08c4d44983223/.tmp/info/ecb8d22b97c74f96bd0475ba204a61ea as hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/data/default/TestLogRolling-testSlowSyncLogRolling/f8a0e31e333efe98bcc08c4d44983223/info/ecb8d22b97c74f96bd0475ba204a61ea
2024-12-11T16:23:26,543 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/data/default/TestLogRolling-testSlowSyncLogRolling/f8a0e31e333efe98bcc08c4d44983223/info/ecb8d22b97c74f96bd0475ba204a61ea, entries=7, sequenceid=42, filesize=12.2 K
2024-12-11T16:23:26,545 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for f8a0e31e333efe98bcc08c4d44983223 in 52ms, sequenceid=42, compaction requested=false
2024-12-11T16:23:26,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f8a0e31e333efe98bcc08c4d44983223:

2024-12-11T16:23:26,546 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=39.3 K, sizeToCheck=16.0 K
2024-12-11T16:23:26,546 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-11T16:23:26,546 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/data/default/TestLogRolling-testSlowSyncLogRolling/f8a0e31e333efe98bcc08c4d44983223/info/59e5482cd032486499ded6e13106beea because midkey is the same as first or last row
2024-12-11T16:23:32,198 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region f8a0e31e333efe98bcc08c4d44983223, had cached 0 bytes from a total of 40219
2024-12-11T16:23:34,514 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-12-11T16:23:34,515 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-11T16:23:34,515 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-11T16:23:34,521 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T16:23:34,521 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-12-11T16:23:34,522 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T16:23:34,522 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-12-11T16:23:34,522 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1718951232, stopped=false
2024-12-11T16:23:34,522 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=7446a65196a1,39377,1733934116649
2024-12-11T16:23:34,533 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43823-0x10015bcaa160001, quorum=127.0.0.1:53474, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-11T16:23:34,533 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39377-0x10015bcaa160000, quorum=127.0.0.1:53474, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-11T16:23:34,533 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43823-0x10015bcaa160001, quorum=127.0.0.1:53474, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:23:34,533 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39377-0x10015bcaa160000, quorum=127.0.0.1:53474, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:23:34,533 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-11T16:23:34,533 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-11T16:23:34,533 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-11T16:23:34,533 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T16:23:34,534 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '7446a65196a1,43823,1733934117800' *****
2024-12-11T16:23:34,534 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-12-11T16:23:34,534 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43823-0x10015bcaa160001, quorum=127.0.0.1:53474, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-11T16:23:34,534 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:39377-0x10015bcaa160000, quorum=127.0.0.1:53474, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-11T16:23:34,534 INFO  [RS:0;7446a65196a1:43823 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-11T16:23:34,534 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-12-11T16:23:34,534 INFO  [RS:0;7446a65196a1:43823 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-11T16:23:34,535 INFO  [RS:0;7446a65196a1:43823 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-11T16:23:34,535 INFO  [RS:0;7446a65196a1:43823 {}] regionserver.HRegionServer(3091): Received CLOSE for f8a0e31e333efe98bcc08c4d44983223
2024-12-11T16:23:34,536 INFO  [RS:0;7446a65196a1:43823 {}] regionserver.HRegionServer(959): stopping server 7446a65196a1,43823,1733934117800
2024-12-11T16:23:34,536 INFO  [RS:0;7446a65196a1:43823 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-11T16:23:34,536 INFO  [RS:0;7446a65196a1:43823 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;7446a65196a1:43823.
2024-12-11T16:23:34,536 DEBUG [RS:0;7446a65196a1:43823 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-11T16:23:34,536 DEBUG [RS:0;7446a65196a1:43823 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T16:23:34,536 DEBUG [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f8a0e31e333efe98bcc08c4d44983223, disabling compactions & flushes
2024-12-11T16:23:34,536 INFO  [RS:0;7446a65196a1:43823 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-11T16:23:34,536 INFO  [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testSlowSyncLogRolling,,1733934121664.f8a0e31e333efe98bcc08c4d44983223.
2024-12-11T16:23:34,536 INFO  [RS:0;7446a65196a1:43823 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-11T16:23:34,536 DEBUG [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testSlowSyncLogRolling,,1733934121664.f8a0e31e333efe98bcc08c4d44983223.
2024-12-11T16:23:34,536 INFO  [RS:0;7446a65196a1:43823 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-11T16:23:34,536 DEBUG [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1733934121664.f8a0e31e333efe98bcc08c4d44983223. after waiting 0 ms
2024-12-11T16:23:34,536 DEBUG [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1733934121664.f8a0e31e333efe98bcc08c4d44983223.
2024-12-11T16:23:34,537 INFO  [RS:0;7446a65196a1:43823 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-12-11T16:23:34,537 INFO  [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing f8a0e31e333efe98bcc08c4d44983223 1/1 column families, dataSize=3.15 KB heapSize=3.63 KB
2024-12-11T16:23:34,537 INFO  [RS:0;7446a65196a1:43823 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-12-11T16:23:34,537 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-11T16:23:34,537 DEBUG [RS:0;7446a65196a1:43823 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, f8a0e31e333efe98bcc08c4d44983223=TestLogRolling-testSlowSyncLogRolling,,1733934121664.f8a0e31e333efe98bcc08c4d44983223.}
2024-12-11T16:23:34,537 INFO  [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-11T16:23:34,537 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-11T16:23:34,537 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-11T16:23:34,537 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-11T16:23:34,538 DEBUG [RS:0;7446a65196a1:43823 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, f8a0e31e333efe98bcc08c4d44983223
2024-12-11T16:23:34,538 INFO  [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.65 KB heapSize=3.67 KB
2024-12-11T16:23:34,550 DEBUG [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/data/default/TestLogRolling-testSlowSyncLogRolling/f8a0e31e333efe98bcc08c4d44983223/.tmp/info/63fd8705602647448d5e93f1236213cf is 1080, key is row0029/info:/1733934208495/Put/seqid=0
2024-12-11T16:23:34,564 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42931 is added to blk_1073741850_1026 (size=8193)
2024-12-11T16:23:34,565 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42705 is added to blk_1073741850_1026 (size=8193)
2024-12-11T16:23:34,566 INFO  [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.15 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/data/default/TestLogRolling-testSlowSyncLogRolling/f8a0e31e333efe98bcc08c4d44983223/.tmp/info/63fd8705602647448d5e93f1236213cf
2024-12-11T16:23:34,587 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/data/hbase/meta/1588230740/.tmp/info/4e07fab4f8914801b27f4eea397ce5ce is 195, key is TestLogRolling-testSlowSyncLogRolling,,1733934121664.f8a0e31e333efe98bcc08c4d44983223./info:regioninfo/1733934122272/Put/seqid=0
2024-12-11T16:23:34,601 DEBUG [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/data/default/TestLogRolling-testSlowSyncLogRolling/f8a0e31e333efe98bcc08c4d44983223/.tmp/info/63fd8705602647448d5e93f1236213cf as hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/data/default/TestLogRolling-testSlowSyncLogRolling/f8a0e31e333efe98bcc08c4d44983223/info/63fd8705602647448d5e93f1236213cf
2024-12-11T16:23:34,629 INFO  [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/data/default/TestLogRolling-testSlowSyncLogRolling/f8a0e31e333efe98bcc08c4d44983223/info/63fd8705602647448d5e93f1236213cf, entries=3, sequenceid=48, filesize=8.0 K
2024-12-11T16:23:34,635 INFO  [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for f8a0e31e333efe98bcc08c4d44983223 in 98ms, sequenceid=48, compaction requested=true
2024-12-11T16:23:34,636 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733934121664.f8a0e31e333efe98bcc08c4d44983223.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/data/default/TestLogRolling-testSlowSyncLogRolling/f8a0e31e333efe98bcc08c4d44983223/info/83ee13b8ffa4461fb2b6f413299d2e69, hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/data/default/TestLogRolling-testSlowSyncLogRolling/f8a0e31e333efe98bcc08c4d44983223/info/e7802e6da06a435d925894016d96ec95, hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/data/default/TestLogRolling-testSlowSyncLogRolling/f8a0e31e333efe98bcc08c4d44983223/info/2aea13df3fba4ce1837ecd226a44bfb7] to archive
2024-12-11T16:23:34,641 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42705 is added to blk_1073741851_1027 (size=7016)
2024-12-11T16:23:34,644 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733934121664.f8a0e31e333efe98bcc08c4d44983223.-1 {}] backup.HFileArchiver(363): Archiving compacted files.
2024-12-11T16:23:34,644 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42931 is added to blk_1073741851_1027 (size=7016)
2024-12-11T16:23:34,646 INFO  [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.45 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/data/hbase/meta/1588230740/.tmp/info/4e07fab4f8914801b27f4eea397ce5ce
2024-12-11T16:23:34,659 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/data/default/TestLogRolling-testSlowSyncLogRolling/f8a0e31e333efe98bcc08c4d44983223/info/e7802e6da06a435d925894016d96ec95 to hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/archive/data/default/TestLogRolling-testSlowSyncLogRolling/f8a0e31e333efe98bcc08c4d44983223/info/e7802e6da06a435d925894016d96ec95
2024-12-11T16:23:34,660 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/data/default/TestLogRolling-testSlowSyncLogRolling/f8a0e31e333efe98bcc08c4d44983223/info/2aea13df3fba4ce1837ecd226a44bfb7 to hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/archive/data/default/TestLogRolling-testSlowSyncLogRolling/f8a0e31e333efe98bcc08c4d44983223/info/2aea13df3fba4ce1837ecd226a44bfb7
2024-12-11T16:23:34,671 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/data/default/TestLogRolling-testSlowSyncLogRolling/f8a0e31e333efe98bcc08c4d44983223/info/83ee13b8ffa4461fb2b6f413299d2e69 to hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/archive/data/default/TestLogRolling-testSlowSyncLogRolling/f8a0e31e333efe98bcc08c4d44983223/info/83ee13b8ffa4461fb2b6f413299d2e69
2024-12-11T16:23:34,685 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733934121664.f8a0e31e333efe98bcc08c4d44983223.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=7446a65196a1:39377 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 16 more
2024-12-11T16:23:34,686 WARN  [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733934121664.f8a0e31e333efe98bcc08c4d44983223.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [83ee13b8ffa4461fb2b6f413299d2e69=12509, e7802e6da06a435d925894016d96ec95=12509, 2aea13df3fba4ce1837ecd226a44bfb7=12509]
2024-12-11T16:23:34,689 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/data/hbase/meta/1588230740/.tmp/ns/9f5af1f321d34acdab59f811fad8d3f6 is 43, key is default/ns:d/1733934121221/Put/seqid=0
2024-12-11T16:23:34,714 DEBUG [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/data/default/TestLogRolling-testSlowSyncLogRolling/f8a0e31e333efe98bcc08c4d44983223/recovered.edits/51.seqid, newMaxSeqId=51, maxSeqId=1
2024-12-11T16:23:34,719 INFO  [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testSlowSyncLogRolling,,1733934121664.f8a0e31e333efe98bcc08c4d44983223.
2024-12-11T16:23:34,720 DEBUG [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f8a0e31e333efe98bcc08c4d44983223:
Waiting for close lock at 1733934214536Running coprocessor pre-close hooks at 1733934214536Disabling compacts and flushes for region at 1733934214536Disabling writes for close at 1733934214536Obtaining lock to block concurrent updates at 1733934214537 (+1 ms)Preparing flush snapshotting stores in f8a0e31e333efe98bcc08c4d44983223 at 1733934214537Finished memstore snapshotting TestLogRolling-testSlowSyncLogRolling,,1733934121664.f8a0e31e333efe98bcc08c4d44983223., syncing WAL and waiting on mvcc, flushsize=dataSize=3228, getHeapSize=3696, getOffHeapSize=0, getCellsCount=3 at 1733934214537Flushing stores of TestLogRolling-testSlowSyncLogRolling,,1733934121664.f8a0e31e333efe98bcc08c4d44983223. at 1733934214539 (+2 ms)Flushing f8a0e31e333efe98bcc08c4d44983223/info: creating writer at 1733934214539Flushing f8a0e31e333efe98bcc08c4d44983223/info: appending metadata at 1733934214550 (+11 ms)Flushing f8a0e31e333efe98bcc08c4d44983223/info: closing flushed file at 1733934214550Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3b04df5e: reopening flushed file at 1733934214596 (+46 ms)Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for f8a0e31e333efe98bcc08c4d44983223 in 98ms, sequenceid=48, compaction requested=true at 1733934214635 (+39 ms)Writing region close event to WAL at 1733934214687 (+52 ms)Running coprocessor post-close hooks at 1733934214717 (+30 ms)Closed at 1733934214719 (+2 ms)
2024-12-11T16:23:34,721 DEBUG [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testSlowSyncLogRolling,,1733934121664.f8a0e31e333efe98bcc08c4d44983223.
2024-12-11T16:23:34,733 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42931 is added to blk_1073741852_1028 (size=5153)
2024-12-11T16:23:34,733 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42705 is added to blk_1073741852_1028 (size=5153)
2024-12-11T16:23:34,738 DEBUG [RS:0;7446a65196a1:43823 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-12-11T16:23:34,739 INFO  [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/data/hbase/meta/1588230740/.tmp/ns/9f5af1f321d34acdab59f811fad8d3f6
2024-12-11T16:23:34,784 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/data/hbase/meta/1588230740/.tmp/table/3661301a6f3c4130b87259fe6b780781 is 73, key is TestLogRolling-testSlowSyncLogRolling/table:state/1733934122298/Put/seqid=0
2024-12-11T16:23:34,803 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42705 is added to blk_1073741853_1029 (size=5396)
2024-12-11T16:23:34,804 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42931 is added to blk_1073741853_1029 (size=5396)
2024-12-11T16:23:34,938 DEBUG [RS:0;7446a65196a1:43823 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-12-11T16:23:35,139 DEBUG [RS:0;7446a65196a1:43823 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-12-11T16:23:35,195 INFO  [regionserver/7446a65196a1:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-12-11T16:23:35,195 INFO  [regionserver/7446a65196a1:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-12-11T16:23:35,202 INFO  [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=138 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/data/hbase/meta/1588230740/.tmp/table/3661301a6f3c4130b87259fe6b780781
2024-12-11T16:23:35,215 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/data/hbase/meta/1588230740/.tmp/info/4e07fab4f8914801b27f4eea397ce5ce as hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/data/hbase/meta/1588230740/info/4e07fab4f8914801b27f4eea397ce5ce
2024-12-11T16:23:35,230 INFO  [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/data/hbase/meta/1588230740/info/4e07fab4f8914801b27f4eea397ce5ce, entries=10, sequenceid=11, filesize=6.9 K
2024-12-11T16:23:35,233 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/data/hbase/meta/1588230740/.tmp/ns/9f5af1f321d34acdab59f811fad8d3f6 as hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/data/hbase/meta/1588230740/ns/9f5af1f321d34acdab59f811fad8d3f6
2024-12-11T16:23:35,244 INFO  [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/data/hbase/meta/1588230740/ns/9f5af1f321d34acdab59f811fad8d3f6, entries=2, sequenceid=11, filesize=5.0 K
2024-12-11T16:23:35,245 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/data/hbase/meta/1588230740/.tmp/table/3661301a6f3c4130b87259fe6b780781 as hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/data/hbase/meta/1588230740/table/3661301a6f3c4130b87259fe6b780781
2024-12-11T16:23:35,254 INFO  [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/data/hbase/meta/1588230740/table/3661301a6f3c4130b87259fe6b780781, entries=2, sequenceid=11, filesize=5.3 K
2024-12-11T16:23:35,255 INFO  [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.65 KB/1692, heapSize ~3.38 KB/3456, currentSize=0 B/0 for 1588230740 in 718ms, sequenceid=11, compaction requested=false
2024-12-11T16:23:35,266 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-12-11T16:23:35,267 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-11T16:23:35,267 INFO  [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-11T16:23:35,267 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733934214537Running coprocessor pre-close hooks at 1733934214537Disabling compacts and flushes for region at 1733934214537Disabling writes for close at 1733934214537Obtaining lock to block concurrent updates at 1733934214538 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1733934214538Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1692, getHeapSize=3696, getOffHeapSize=0, getCellsCount=14 at 1733934214538Flushing stores of hbase:meta,,1.1588230740 at 1733934214542 (+4 ms)Flushing 1588230740/info: creating writer at 1733934214542Flushing 1588230740/info: appending metadata at 1733934214586 (+44 ms)Flushing 1588230740/info: closing flushed file at 1733934214586Flushing 1588230740/ns: creating writer at 1733934214668 (+82 ms)Flushing 1588230740/ns: appending metadata at 1733934214689 (+21 ms)Flushing 1588230740/ns: closing flushed file at 1733934214689Flushing 1588230740/table: creating writer at 1733934214751 (+62 ms)Flushing 1588230740/table: appending metadata at 1733934214784 (+33 ms)Flushing 1588230740/table: closing flushed file at 1733934214784Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5b06d58d: reopening flushed file at 1733934215214 (+430 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@26c4022e: reopening flushed file at 1733934215230 (+16 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@664679d5: reopening flushed file at 1733934215244 (+14 ms)Finished flush of dataSize ~1.65 KB/1692, heapSize ~3.38 KB/3456, currentSize=0 B/0 for 1588230740 in 718ms, sequenceid=11, compaction requested=false at 1733934215256 (+12 ms)Writing region close event to WAL at 1733934215262 (+6 ms)Running coprocessor post-close hooks at 1733934215267 (+5 ms)Closed at 1733934215267
2024-12-11T16:23:35,267 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-12-11T16:23:35,339 INFO  [RS:0;7446a65196a1:43823 {}] regionserver.HRegionServer(976): stopping server 7446a65196a1,43823,1733934117800; all regions closed.
2024-12-11T16:23:35,341 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:23:35,341 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:23:35,341 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:23:35,341 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:23:35,341 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:23:35,349 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42705 is added to blk_1073741834_1010 (size=3066)
2024-12-11T16:23:35,349 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42931 is added to blk_1073741834_1010 (size=3066)
2024-12-11T16:23:35,754 DEBUG [RS:0;7446a65196a1:43823 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/oldWALs
2024-12-11T16:23:35,754 INFO  [RS:0;7446a65196a1:43823 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 7446a65196a1%2C43823%2C1733934117800.meta:.meta(num 1733934120891)
2024-12-11T16:23:35,755 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:23:35,755 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:23:35,757 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:23:35,757 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:23:35,757 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:23:35,760 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42931 is added to blk_1073741847_1023 (size=12695)
2024-12-11T16:23:35,760 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42705 is added to blk_1073741847_1023 (size=12695)
2024-12-11T16:23:35,764 DEBUG [RS:0;7446a65196a1:43823 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/oldWALs
2024-12-11T16:23:35,764 INFO  [RS:0;7446a65196a1:43823 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 7446a65196a1%2C43823%2C1733934117800:(num 1733934194465)
2024-12-11T16:23:35,765 DEBUG [RS:0;7446a65196a1:43823 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T16:23:35,765 INFO  [RS:0;7446a65196a1:43823 {}] regionserver.LeaseManager(133): Closed leases
2024-12-11T16:23:35,765 INFO  [RS:0;7446a65196a1:43823 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-11T16:23:35,765 INFO  [RS:0;7446a65196a1:43823 {}] hbase.ChoreService(370): Chore service for: regionserver/7446a65196a1:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-12-11T16:23:35,765 INFO  [RS:0;7446a65196a1:43823 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-11T16:23:35,765 INFO  [regionserver/7446a65196a1:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-11T16:23:35,766 INFO  [RS:0;7446a65196a1:43823 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43823
2024-12-11T16:23:35,807 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43823-0x10015bcaa160001, quorum=127.0.0.1:53474, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/7446a65196a1,43823,1733934117800
2024-12-11T16:23:35,807 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39377-0x10015bcaa160000, quorum=127.0.0.1:53474, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-11T16:23:35,807 INFO  [RS:0;7446a65196a1:43823 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-11T16:23:35,808 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [7446a65196a1,43823,1733934117800]
2024-12-11T16:23:35,834 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/7446a65196a1,43823,1733934117800 already deleted, retry=false
2024-12-11T16:23:35,835 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 7446a65196a1,43823,1733934117800 expired; onlineServers=0
2024-12-11T16:23:35,835 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '7446a65196a1,39377,1733934116649' *****
2024-12-11T16:23:35,835 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-12-11T16:23:35,835 INFO  [M:0;7446a65196a1:39377 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-11T16:23:35,835 INFO  [M:0;7446a65196a1:39377 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-11T16:23:35,835 DEBUG [M:0;7446a65196a1:39377 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-12-11T16:23:35,836 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-12-11T16:23:35,836 DEBUG [M:0;7446a65196a1:39377 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-12-11T16:23:35,836 DEBUG [master/7446a65196a1:0:becomeActiveMaster-HFileCleaner.large.0-1733934119697 {}] cleaner.HFileCleaner(306): Exit Thread[master/7446a65196a1:0:becomeActiveMaster-HFileCleaner.large.0-1733934119697,5,FailOnTimeoutGroup]
2024-12-11T16:23:35,836 DEBUG [master/7446a65196a1:0:becomeActiveMaster-HFileCleaner.small.0-1733934119707 {}] cleaner.HFileCleaner(306): Exit Thread[master/7446a65196a1:0:becomeActiveMaster-HFileCleaner.small.0-1733934119707,5,FailOnTimeoutGroup]
2024-12-11T16:23:35,836 INFO  [M:0;7446a65196a1:39377 {}] hbase.ChoreService(370): Chore service for: master/7446a65196a1:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-12-11T16:23:35,836 INFO  [M:0;7446a65196a1:39377 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-11T16:23:35,836 DEBUG [M:0;7446a65196a1:39377 {}] master.HMaster(1795): Stopping service threads
2024-12-11T16:23:35,836 INFO  [M:0;7446a65196a1:39377 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-12-11T16:23:35,836 INFO  [M:0;7446a65196a1:39377 {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-11T16:23:35,837 INFO  [M:0;7446a65196a1:39377 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-12-11T16:23:35,837 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-12-11T16:23:35,849 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39377-0x10015bcaa160000, quorum=127.0.0.1:53474, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-12-11T16:23:35,849 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39377-0x10015bcaa160000, quorum=127.0.0.1:53474, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:23:35,849 DEBUG [M:0;7446a65196a1:39377 {}] zookeeper.ZKUtil(347): master:39377-0x10015bcaa160000, quorum=127.0.0.1:53474, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-12-11T16:23:35,849 WARN  [M:0;7446a65196a1:39377 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-12-11T16:23:35,850 INFO  [M:0;7446a65196a1:39377 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/.lastflushedseqids
2024-12-11T16:23:35,861 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42931 is added to blk_1073741854_1030 (size=130)
2024-12-11T16:23:35,861 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42705 is added to blk_1073741854_1030 (size=130)
2024-12-11T16:23:35,862 INFO  [M:0;7446a65196a1:39377 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-12-11T16:23:35,862 INFO  [M:0;7446a65196a1:39377 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-12-11T16:23:35,862 DEBUG [M:0;7446a65196a1:39377 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-11T16:23:35,862 INFO  [M:0;7446a65196a1:39377 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T16:23:35,862 DEBUG [M:0;7446a65196a1:39377 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T16:23:35,862 DEBUG [M:0;7446a65196a1:39377 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-11T16:23:35,862 DEBUG [M:0;7446a65196a1:39377 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T16:23:35,862 INFO  [M:0;7446a65196a1:39377 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.02 KB heapSize=29.20 KB
2024-12-11T16:23:35,879 DEBUG [M:0;7446a65196a1:39377 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8d7605775c3145de8a17d4fb311ce89d is 82, key is hbase:meta,,1/info:regioninfo/1733934121053/Put/seqid=0
2024-12-11T16:23:35,886 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42931 is added to blk_1073741855_1031 (size=5672)
2024-12-11T16:23:35,887 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42705 is added to blk_1073741855_1031 (size=5672)
2024-12-11T16:23:35,887 INFO  [M:0;7446a65196a1:39377 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8d7605775c3145de8a17d4fb311ce89d
2024-12-11T16:23:35,916 DEBUG [M:0;7446a65196a1:39377 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/181cbce4242f482b8a66b86cbc100e6e is 766, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733934122308/Put/seqid=0
2024-12-11T16:23:35,922 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42705 is added to blk_1073741856_1032 (size=6247)
2024-12-11T16:23:35,923 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42931 is added to blk_1073741856_1032 (size=6247)
2024-12-11T16:23:35,927 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43823-0x10015bcaa160001, quorum=127.0.0.1:53474, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-11T16:23:35,927 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43823-0x10015bcaa160001, quorum=127.0.0.1:53474, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-11T16:23:35,927 INFO  [RS:0;7446a65196a1:43823 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-11T16:23:35,927 INFO  [RS:0;7446a65196a1:43823 {}] regionserver.HRegionServer(1031): Exiting; stopping=7446a65196a1,43823,1733934117800; zookeeper connection closed.
2024-12-11T16:23:35,927 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@9dff3eb {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@9dff3eb
2024-12-11T16:23:35,928 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-12-11T16:23:36,198 INFO  [regionserver/7446a65196a1:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-11T16:23:36,323 INFO  [M:0;7446a65196a1:39377 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.42 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/181cbce4242f482b8a66b86cbc100e6e
2024-12-11T16:23:36,330 INFO  [M:0;7446a65196a1:39377 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 181cbce4242f482b8a66b86cbc100e6e
2024-12-11T16:23:36,350 DEBUG [M:0;7446a65196a1:39377 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/92a5fe99a5ca4ee4a07166c76c930b2f is 69, key is 7446a65196a1,43823,1733934117800/rs:state/1733934119903/Put/seqid=0
2024-12-11T16:23:36,357 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42705 is added to blk_1073741857_1033 (size=5156)
2024-12-11T16:23:36,359 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42931 is added to blk_1073741857_1033 (size=5156)
2024-12-11T16:23:36,363 INFO  [M:0;7446a65196a1:39377 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/92a5fe99a5ca4ee4a07166c76c930b2f
2024-12-11T16:23:36,387 DEBUG [M:0;7446a65196a1:39377 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/5fafb3c73f9044009864eb856beb7852 is 52, key is load_balancer_on/state:d/1733934121625/Put/seqid=0
2024-12-11T16:23:36,394 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42705 is added to blk_1073741858_1034 (size=5056)
2024-12-11T16:23:36,394 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42931 is added to blk_1073741858_1034 (size=5056)
2024-12-11T16:23:36,395 INFO  [M:0;7446a65196a1:39377 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/5fafb3c73f9044009864eb856beb7852
2024-12-11T16:23:36,411 DEBUG [M:0;7446a65196a1:39377 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8d7605775c3145de8a17d4fb311ce89d as hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/8d7605775c3145de8a17d4fb311ce89d
2024-12-11T16:23:36,423 INFO  [M:0;7446a65196a1:39377 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/8d7605775c3145de8a17d4fb311ce89d, entries=8, sequenceid=59, filesize=5.5 K
2024-12-11T16:23:36,425 DEBUG [M:0;7446a65196a1:39377 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/181cbce4242f482b8a66b86cbc100e6e as hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/181cbce4242f482b8a66b86cbc100e6e
2024-12-11T16:23:36,439 INFO  [M:0;7446a65196a1:39377 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 181cbce4242f482b8a66b86cbc100e6e
2024-12-11T16:23:36,439 INFO  [M:0;7446a65196a1:39377 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/181cbce4242f482b8a66b86cbc100e6e, entries=6, sequenceid=59, filesize=6.1 K
2024-12-11T16:23:36,442 DEBUG [M:0;7446a65196a1:39377 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/92a5fe99a5ca4ee4a07166c76c930b2f as hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/92a5fe99a5ca4ee4a07166c76c930b2f
2024-12-11T16:23:36,453 INFO  [M:0;7446a65196a1:39377 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/92a5fe99a5ca4ee4a07166c76c930b2f, entries=1, sequenceid=59, filesize=5.0 K
2024-12-11T16:23:36,455 DEBUG [M:0;7446a65196a1:39377 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/5fafb3c73f9044009864eb856beb7852 as hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/5fafb3c73f9044009864eb856beb7852
2024-12-11T16:23:36,465 INFO  [M:0;7446a65196a1:39377 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/5fafb3c73f9044009864eb856beb7852, entries=1, sequenceid=59, filesize=4.9 K
2024-12-11T16:23:36,467 INFO  [M:0;7446a65196a1:39377 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.02 KB/23576, heapSize ~29.14 KB/29840, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 605ms, sequenceid=59, compaction requested=false
2024-12-11T16:23:36,473 INFO  [M:0;7446a65196a1:39377 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T16:23:36,473 DEBUG [M:0;7446a65196a1:39377 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733934215862Disabling compacts and flushes for region at 1733934215862Disabling writes for close at 1733934215862Obtaining lock to block concurrent updates at 1733934215862Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733934215862Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23576, getHeapSize=29840, getOffHeapSize=0, getCellsCount=70 at 1733934215863 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733934215864 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733934215864Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733934215878 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733934215878Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733934215895 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733934215916 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733934215916Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733934216330 (+414 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733934216349 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733934216349Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733934216371 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733934216387 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733934216387Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@a8046fa: reopening flushed file at 1733934216404 (+17 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@22d4f193: reopening flushed file at 1733934216424 (+20 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5fa6dc3: reopening flushed file at 1733934216439 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@c8a8a41: reopening flushed file at 1733934216453 (+14 ms)Finished flush of dataSize ~23.02 KB/23576, heapSize ~29.14 KB/29840, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 605ms, sequenceid=59, compaction requested=false at 1733934216467 (+14 ms)Writing region close event to WAL at 1733934216473 (+6 ms)Closed at 1733934216473
2024-12-11T16:23:36,474 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:23:36,474 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:23:36,474 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:23:36,475 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:23:36,475 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:23:36,477 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42931 is added to blk_1073741830_1006 (size=27973)
2024-12-11T16:23:36,479 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42705 is added to blk_1073741830_1006 (size=27973)
2024-12-11T16:23:36,481 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-11T16:23:36,481 INFO  [M:0;7446a65196a1:39377 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-12-11T16:23:36,481 INFO  [M:0;7446a65196a1:39377 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39377
2024-12-11T16:23:36,481 INFO  [M:0;7446a65196a1:39377 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-11T16:23:36,643 INFO  [M:0;7446a65196a1:39377 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-11T16:23:36,643 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39377-0x10015bcaa160000, quorum=127.0.0.1:53474, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-11T16:23:36,643 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39377-0x10015bcaa160000, quorum=127.0.0.1:53474, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-11T16:23:36,649 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@590b36b7{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T16:23:36,652 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@237fc06a{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-11T16:23:36,652 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-11T16:23:36,652 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@35f1cf70{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-11T16:23:36,652 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@43794ae7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/762be7fd-a69f-281d-5246-84312cc22b69/hadoop.log.dir/,STOPPED}
2024-12-11T16:23:36,658 WARN  [BP-1818873448-172.17.0.2-1733934111750 heartbeating to localhost/127.0.0.1:36921 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-11T16:23:36,658 WARN  [BP-1818873448-172.17.0.2-1733934111750 heartbeating to localhost/127.0.0.1:36921 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1818873448-172.17.0.2-1733934111750 (Datanode Uuid 257da275-4cb7-4478-9896-000aa2c9722a) service to localhost/127.0.0.1:36921
2024-12-11T16:23:36,660 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/762be7fd-a69f-281d-5246-84312cc22b69/cluster_8f17652b-4025-2fa7-ad27-5fb00f5849ad/data/data4/current/BP-1818873448-172.17.0.2-1733934111750 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T16:23:36,663 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-11T16:23:36,663 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/762be7fd-a69f-281d-5246-84312cc22b69/cluster_8f17652b-4025-2fa7-ad27-5fb00f5849ad/data/data3/current/BP-1818873448-172.17.0.2-1733934111750 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T16:23:36,663 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-11T16:23:36,664 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-11T16:23:36,670 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@11e88411{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T16:23:36,670 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@75423500{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-11T16:23:36,670 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-11T16:23:36,671 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@621a7cbc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-11T16:23:36,671 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@26c88bf4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/762be7fd-a69f-281d-5246-84312cc22b69/hadoop.log.dir/,STOPPED}
2024-12-11T16:23:36,680 WARN  [BP-1818873448-172.17.0.2-1733934111750 heartbeating to localhost/127.0.0.1:36921 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-11T16:23:36,680 WARN  [BP-1818873448-172.17.0.2-1733934111750 heartbeating to localhost/127.0.0.1:36921 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1818873448-172.17.0.2-1733934111750 (Datanode Uuid 8ba32a4e-b51a-4d2f-8424-e054183c8047) service to localhost/127.0.0.1:36921
2024-12-11T16:23:36,681 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/762be7fd-a69f-281d-5246-84312cc22b69/cluster_8f17652b-4025-2fa7-ad27-5fb00f5849ad/data/data1/current/BP-1818873448-172.17.0.2-1733934111750 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T16:23:36,681 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/762be7fd-a69f-281d-5246-84312cc22b69/cluster_8f17652b-4025-2fa7-ad27-5fb00f5849ad/data/data2/current/BP-1818873448-172.17.0.2-1733934111750 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T16:23:36,681 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-11T16:23:36,681 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-11T16:23:36,681 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-11T16:23:36,697 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6de997b9{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-11T16:23:36,699 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7a0da00a{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-11T16:23:36,699 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-11T16:23:36,700 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@380b8195{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-11T16:23:36,700 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3aee6cb7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/762be7fd-a69f-281d-5246-84312cc22b69/hadoop.log.dir/,STOPPED}
2024-12-11T16:23:36,725 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-12-11T16:23:36,765 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-12-11T16:23:36,773 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testSlowSyncLogRolling Thread=80 (was 12)
Potentially hanging thread: nioEventLoopGroup-4-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:36921
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: HMaster-EventLoopGroup-1-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Async-Client-Retry-Timer-pool-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-2-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-3-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-5-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: weak-ref-cleaner-strictcontextstorage
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176)
	app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: sync.1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: SessionTracker
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163)

Potentially hanging thread: nioEventLoopGroup-5-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: sync.3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: regionserver/7446a65196a1:0.procedureResultReporter
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75)

Potentially hanging thread: sync.1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: Monitor thread for TaskMonitor
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (548703531) connection to localhost/127.0.0.1:36921 from jenkins.hfs.0
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: GcTimeMonitor obsWindow = 60000, sleepInterval = 5000, maxGcTimePerc = 100
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.util.GcTimeMonitor.run(GcTimeMonitor.java:161)

Potentially hanging thread: master/7446a65196a1:0:becomeActiveMaster-MemStoreChunkPool Statistics
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-5-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Timer for 'HBase' metrics system
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: ForkJoinPool-2-worker-2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: ForkJoinPool-2-worker-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: RpcClient-timer-pool-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:36921
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-3
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-3-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: IPC Client (548703531) connection to localhost/127.0.0.1:36921 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:36921
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-3-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: SSL Certificates Store Monitor
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.Object.wait(Object.java:338)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:537)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: region-location-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-2-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-2-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (548703531) connection to localhost/127.0.0.1:36921 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: sync.1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Time-limited test.named-queue-events-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47)
	app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56)
	app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159)
	app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HBase-Metrics2-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: SnapshotHandlerChoreCleaner
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:36921
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-4-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.0@localhost:36921
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176)
	app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: master/7446a65196a1:0:becomeActiveMaster-MemStoreChunkPool Statistics
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-1-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-1-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-1
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: org.apache.hadoop.hdfs.PeerCache@56ce92cd
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:253)
	app//org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46)
	app//org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-4-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=404 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=893 (was 1185), ProcessCount=11 (was 11), AvailableMemoryMB=1330 (was 2027)
2024-12-11T16:23:36,780 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnDatanodeDeath Thread=81, OpenFileDescriptor=404, MaxFileDescriptor=1048576, SystemLoadAverage=893, ProcessCount=11, AvailableMemoryMB=1330
2024-12-11T16:23:36,780 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-12-11T16:23:36,781 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/762be7fd-a69f-281d-5246-84312cc22b69/hadoop.log.dir so I do NOT create it in target/test-data/9849ee79-554d-04b7-b255-e4857b767920
2024-12-11T16:23:36,781 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/762be7fd-a69f-281d-5246-84312cc22b69/hadoop.tmp.dir so I do NOT create it in target/test-data/9849ee79-554d-04b7-b255-e4857b767920
2024-12-11T16:23:36,781 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9849ee79-554d-04b7-b255-e4857b767920/cluster_b1b3091a-5088-7047-daf8-a83f640e6f09, deleteOnExit=true
2024-12-11T16:23:36,781 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-12-11T16:23:36,781 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9849ee79-554d-04b7-b255-e4857b767920/test.cache.data in system properties and HBase conf
2024-12-11T16:23:36,781 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9849ee79-554d-04b7-b255-e4857b767920/hadoop.tmp.dir in system properties and HBase conf
2024-12-11T16:23:36,781 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9849ee79-554d-04b7-b255-e4857b767920/hadoop.log.dir in system properties and HBase conf
2024-12-11T16:23:36,781 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9849ee79-554d-04b7-b255-e4857b767920/mapreduce.cluster.local.dir in system properties and HBase conf
2024-12-11T16:23:36,781 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9849ee79-554d-04b7-b255-e4857b767920/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-12-11T16:23:36,781 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-12-11T16:23:36,782 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-11T16:23:36,782 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9849ee79-554d-04b7-b255-e4857b767920/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-12-11T16:23:36,782 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9849ee79-554d-04b7-b255-e4857b767920/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-12-11T16:23:36,782 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9849ee79-554d-04b7-b255-e4857b767920/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-12-11T16:23:36,782 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9849ee79-554d-04b7-b255-e4857b767920/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-11T16:23:36,782 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9849ee79-554d-04b7-b255-e4857b767920/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-12-11T16:23:36,782 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9849ee79-554d-04b7-b255-e4857b767920/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-12-11T16:23:36,782 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9849ee79-554d-04b7-b255-e4857b767920/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-11T16:23:36,782 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9849ee79-554d-04b7-b255-e4857b767920/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-11T16:23:36,782 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9849ee79-554d-04b7-b255-e4857b767920/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-12-11T16:23:36,783 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9849ee79-554d-04b7-b255-e4857b767920/nfs.dump.dir in system properties and HBase conf
2024-12-11T16:23:36,783 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9849ee79-554d-04b7-b255-e4857b767920/java.io.tmpdir in system properties and HBase conf
2024-12-11T16:23:36,783 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9849ee79-554d-04b7-b255-e4857b767920/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-11T16:23:36,783 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9849ee79-554d-04b7-b255-e4857b767920/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-12-11T16:23:36,783 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9849ee79-554d-04b7-b255-e4857b767920/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-12-11T16:23:36,796 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-11T16:23:37,165 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-11T16:23:37,173 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-11T16:23:37,186 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-11T16:23:37,186 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-11T16:23:37,186 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-11T16:23:37,187 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-11T16:23:37,188 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@51945247{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9849ee79-554d-04b7-b255-e4857b767920/hadoop.log.dir/,AVAILABLE}
2024-12-11T16:23:37,189 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5ee8b880{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-11T16:23:37,192 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-11T16:23:37,192 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-12-11T16:23:37,193 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-12-11T16:23:37,194 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling
2024-12-11T16:23:37,306 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5b8ba39e{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9849ee79-554d-04b7-b255-e4857b767920/java.io.tmpdir/jetty-localhost-37485-hadoop-hdfs-3_4_1-tests_jar-_-any-8394721834811362059/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-11T16:23:37,307 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2d2a6b1f{HTTP/1.1, (http/1.1)}{localhost:37485}
2024-12-11T16:23:37,307 INFO  [Time-limited test {}] server.Server(415): Started @107977ms
2024-12-11T16:23:37,323 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-11T16:23:37,650 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-11T16:23:37,658 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-11T16:23:37,663 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-11T16:23:37,663 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-11T16:23:37,663 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-11T16:23:37,665 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4e62fea{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9849ee79-554d-04b7-b255-e4857b767920/hadoop.log.dir/,AVAILABLE}
2024-12-11T16:23:37,665 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5383b5e5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-11T16:23:37,809 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@21d2e33a{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9849ee79-554d-04b7-b255-e4857b767920/java.io.tmpdir/jetty-localhost-34889-hadoop-hdfs-3_4_1-tests_jar-_-any-874353571117118017/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T16:23:37,811 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@43aec6f9{HTTP/1.1, (http/1.1)}{localhost:34889}
2024-12-11T16:23:37,811 INFO  [Time-limited test {}] server.Server(415): Started @108481ms
2024-12-11T16:23:37,813 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-11T16:23:37,906 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-11T16:23:37,911 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-11T16:23:37,919 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-11T16:23:37,920 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-11T16:23:37,920 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-11T16:23:37,923 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5d562032{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9849ee79-554d-04b7-b255-e4857b767920/hadoop.log.dir/,AVAILABLE}
2024-12-11T16:23:37,924 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@25ae6aaf{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-11T16:23:38,069 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@47440849{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9849ee79-554d-04b7-b255-e4857b767920/java.io.tmpdir/jetty-localhost-33631-hadoop-hdfs-3_4_1-tests_jar-_-any-9107236948190105014/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T16:23:38,070 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@41a23710{HTTP/1.1, (http/1.1)}{localhost:33631}
2024-12-11T16:23:38,070 INFO  [Time-limited test {}] server.Server(415): Started @108740ms
2024-12-11T16:23:38,072 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-11T16:23:38,705 WARN  [Thread-451 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9849ee79-554d-04b7-b255-e4857b767920/cluster_b1b3091a-5088-7047-daf8-a83f640e6f09/data/data1/current/BP-1424680021-172.17.0.2-1733934216809/current, will proceed with Du for space computation calculation, 
2024-12-11T16:23:38,705 WARN  [Thread-452 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9849ee79-554d-04b7-b255-e4857b767920/cluster_b1b3091a-5088-7047-daf8-a83f640e6f09/data/data2/current/BP-1424680021-172.17.0.2-1733934216809/current, will proceed with Du for space computation calculation, 
2024-12-11T16:23:38,740 WARN  [Thread-416 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-11T16:23:38,743 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xbbc41332b08ea55b with lease ID 0xe402acc167b2365e: Processing first storage report for DS-215ef1b1-569f-4d58-85de-be75269a1585 from datanode DatanodeRegistration(127.0.0.1:46841, datanodeUuid=f150b863-5e60-4a1e-b798-b37dae44fa90, infoPort=44977, infoSecurePort=0, ipcPort=36929, storageInfo=lv=-57;cid=testClusterID;nsid=600264350;c=1733934216809)
2024-12-11T16:23:38,743 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbbc41332b08ea55b with lease ID 0xe402acc167b2365e: from storage DS-215ef1b1-569f-4d58-85de-be75269a1585 node DatanodeRegistration(127.0.0.1:46841, datanodeUuid=f150b863-5e60-4a1e-b798-b37dae44fa90, infoPort=44977, infoSecurePort=0, ipcPort=36929, storageInfo=lv=-57;cid=testClusterID;nsid=600264350;c=1733934216809), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-11T16:23:38,743 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xbbc41332b08ea55b with lease ID 0xe402acc167b2365e: Processing first storage report for DS-a6157f18-2b14-43f6-8a17-2b2654b646d4 from datanode DatanodeRegistration(127.0.0.1:46841, datanodeUuid=f150b863-5e60-4a1e-b798-b37dae44fa90, infoPort=44977, infoSecurePort=0, ipcPort=36929, storageInfo=lv=-57;cid=testClusterID;nsid=600264350;c=1733934216809)
2024-12-11T16:23:38,743 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbbc41332b08ea55b with lease ID 0xe402acc167b2365e: from storage DS-a6157f18-2b14-43f6-8a17-2b2654b646d4 node DatanodeRegistration(127.0.0.1:46841, datanodeUuid=f150b863-5e60-4a1e-b798-b37dae44fa90, infoPort=44977, infoSecurePort=0, ipcPort=36929, storageInfo=lv=-57;cid=testClusterID;nsid=600264350;c=1733934216809), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-11T16:23:38,909 WARN  [Thread-463 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9849ee79-554d-04b7-b255-e4857b767920/cluster_b1b3091a-5088-7047-daf8-a83f640e6f09/data/data3/current/BP-1424680021-172.17.0.2-1733934216809/current, will proceed with Du for space computation calculation, 
2024-12-11T16:23:38,909 WARN  [Thread-464 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9849ee79-554d-04b7-b255-e4857b767920/cluster_b1b3091a-5088-7047-daf8-a83f640e6f09/data/data4/current/BP-1424680021-172.17.0.2-1733934216809/current, will proceed with Du for space computation calculation, 
2024-12-11T16:23:38,930 WARN  [Thread-439 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-11T16:23:38,932 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xdfd889e943f5bab0 with lease ID 0xe402acc167b2365f: Processing first storage report for DS-de6e0e49-d654-4dbe-a963-6b373c7329b3 from datanode DatanodeRegistration(127.0.0.1:39925, datanodeUuid=0ac6d820-d9ff-4f11-892b-d93f697edbd9, infoPort=38513, infoSecurePort=0, ipcPort=40867, storageInfo=lv=-57;cid=testClusterID;nsid=600264350;c=1733934216809)
2024-12-11T16:23:38,932 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xdfd889e943f5bab0 with lease ID 0xe402acc167b2365f: from storage DS-de6e0e49-d654-4dbe-a963-6b373c7329b3 node DatanodeRegistration(127.0.0.1:39925, datanodeUuid=0ac6d820-d9ff-4f11-892b-d93f697edbd9, infoPort=38513, infoSecurePort=0, ipcPort=40867, storageInfo=lv=-57;cid=testClusterID;nsid=600264350;c=1733934216809), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-11T16:23:38,932 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xdfd889e943f5bab0 with lease ID 0xe402acc167b2365f: Processing first storage report for DS-79dcc49f-960b-400d-b295-a2690fa24422 from datanode DatanodeRegistration(127.0.0.1:39925, datanodeUuid=0ac6d820-d9ff-4f11-892b-d93f697edbd9, infoPort=38513, infoSecurePort=0, ipcPort=40867, storageInfo=lv=-57;cid=testClusterID;nsid=600264350;c=1733934216809)
2024-12-11T16:23:38,932 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xdfd889e943f5bab0 with lease ID 0xe402acc167b2365f: from storage DS-79dcc49f-960b-400d-b295-a2690fa24422 node DatanodeRegistration(127.0.0.1:39925, datanodeUuid=0ac6d820-d9ff-4f11-892b-d93f697edbd9, infoPort=38513, infoSecurePort=0, ipcPort=40867, storageInfo=lv=-57;cid=testClusterID;nsid=600264350;c=1733934216809), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-11T16:23:38,980 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9849ee79-554d-04b7-b255-e4857b767920
2024-12-11T16:23:38,985 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9849ee79-554d-04b7-b255-e4857b767920/cluster_b1b3091a-5088-7047-daf8-a83f640e6f09/zookeeper_0, clientPort=50343, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9849ee79-554d-04b7-b255-e4857b767920/cluster_b1b3091a-5088-7047-daf8-a83f640e6f09/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9849ee79-554d-04b7-b255-e4857b767920/cluster_b1b3091a-5088-7047-daf8-a83f640e6f09/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-11T16:23:38,986 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=50343
2024-12-11T16:23:38,987 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T16:23:38,988 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T16:23:39,047 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39925 is added to blk_1073741825_1001 (size=7)
2024-12-11T16:23:39,048 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46841 is added to blk_1073741825_1001 (size=7)
2024-12-11T16:23:39,449 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:34957/user/jenkins/test-data/5bdccf7f-28dc-94ce-526d-9d4d9920011a with version=8
2024-12-11T16:23:39,449 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/hbase-staging
2024-12-11T16:23:39,451 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/7446a65196a1:0 server-side Connection retries=45
2024-12-11T16:23:39,452 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-11T16:23:39,452 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-11T16:23:39,452 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-11T16:23:39,452 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-11T16:23:39,452 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-11T16:23:39,452 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-12-11T16:23:39,452 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-11T16:23:39,453 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35575
2024-12-11T16:23:39,454 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:35575 connecting to ZooKeeper ensemble=127.0.0.1:50343
2024-12-11T16:23:39,516 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:355750x0, quorum=127.0.0.1:50343, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-11T16:23:39,516 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:35575-0x10015be3d850000 connected
2024-12-11T16:23:39,587 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T16:23:39,589 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T16:23:39,602 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35575-0x10015be3d850000, quorum=127.0.0.1:50343, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-11T16:23:39,603 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:34957/user/jenkins/test-data/5bdccf7f-28dc-94ce-526d-9d4d9920011a, hbase.cluster.distributed=false
2024-12-11T16:23:39,605 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35575-0x10015be3d850000, quorum=127.0.0.1:50343, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-11T16:23:39,607 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35575
2024-12-11T16:23:39,611 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35575
2024-12-11T16:23:39,619 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35575
2024-12-11T16:23:39,621 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35575
2024-12-11T16:23:39,621 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35575
2024-12-11T16:23:39,644 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/7446a65196a1:0 server-side Connection retries=45
2024-12-11T16:23:39,644 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-11T16:23:39,644 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-11T16:23:39,644 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-11T16:23:39,644 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-11T16:23:39,644 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-11T16:23:39,644 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-11T16:23:39,644 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-11T16:23:39,647 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41351
2024-12-11T16:23:39,649 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41351 connecting to ZooKeeper ensemble=127.0.0.1:50343
2024-12-11T16:23:39,650 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T16:23:39,654 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T16:23:39,704 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:413510x0, quorum=127.0.0.1:50343, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-11T16:23:39,705 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41351-0x10015be3d850001 connected
2024-12-11T16:23:39,705 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41351-0x10015be3d850001, quorum=127.0.0.1:50343, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-11T16:23:39,706 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-11T16:23:39,707 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-11T16:23:39,712 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41351-0x10015be3d850001, quorum=127.0.0.1:50343, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-11T16:23:39,716 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41351-0x10015be3d850001, quorum=127.0.0.1:50343, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-11T16:23:39,721 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:23:39,722 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41351
2024-12-11T16:23:39,723 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41351
2024-12-11T16:23:39,726 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:23:39,743 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41351
2024-12-11T16:23:39,756 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41351
2024-12-11T16:23:39,756 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41351
2024-12-11T16:23:39,784 DEBUG [M:0;7446a65196a1:35575 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;7446a65196a1:35575
2024-12-11T16:23:39,791 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/7446a65196a1,35575,1733934219451
2024-12-11T16:23:39,803 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41351-0x10015be3d850001, quorum=127.0.0.1:50343, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-11T16:23:39,803 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35575-0x10015be3d850000, quorum=127.0.0.1:50343, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-11T16:23:39,809 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35575-0x10015be3d850000, quorum=127.0.0.1:50343, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/7446a65196a1,35575,1733934219451
2024-12-11T16:23:39,822 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35575-0x10015be3d850000, quorum=127.0.0.1:50343, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:23:39,823 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41351-0x10015be3d850001, quorum=127.0.0.1:50343, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-11T16:23:39,823 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41351-0x10015be3d850001, quorum=127.0.0.1:50343, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:23:39,829 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35575-0x10015be3d850000, quorum=127.0.0.1:50343, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-11T16:23:39,831 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/7446a65196a1,35575,1733934219451 from backup master directory
2024-12-11T16:23:39,841 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35575-0x10015be3d850000, quorum=127.0.0.1:50343, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/7446a65196a1,35575,1733934219451
2024-12-11T16:23:39,841 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41351-0x10015be3d850001, quorum=127.0.0.1:50343, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-11T16:23:39,841 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35575-0x10015be3d850000, quorum=127.0.0.1:50343, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-11T16:23:39,842 WARN  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-11T16:23:39,842 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=7446a65196a1,35575,1733934219451
2024-12-11T16:23:39,861 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:34957/user/jenkins/test-data/5bdccf7f-28dc-94ce-526d-9d4d9920011a/hbase.id] with ID: c5ba6bf3-5a92-4957-b073-9acdde23983a
2024-12-11T16:23:39,861 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:34957/user/jenkins/test-data/5bdccf7f-28dc-94ce-526d-9d4d9920011a/.tmp/hbase.id
2024-12-11T16:23:39,939 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39925 is added to blk_1073741826_1002 (size=42)
2024-12-11T16:23:39,940 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46841 is added to blk_1073741826_1002 (size=42)
2024-12-11T16:23:39,955 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:34957/user/jenkins/test-data/5bdccf7f-28dc-94ce-526d-9d4d9920011a/.tmp/hbase.id]:[hdfs://localhost:34957/user/jenkins/test-data/5bdccf7f-28dc-94ce-526d-9d4d9920011a/hbase.id]
2024-12-11T16:23:40,003 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T16:23:40,003 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-12-11T16:23:40,006 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 3ms.
2024-12-11T16:23:40,041 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35575-0x10015be3d850000, quorum=127.0.0.1:50343, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:23:40,041 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41351-0x10015be3d850001, quorum=127.0.0.1:50343, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:23:40,100 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46841 is added to blk_1073741827_1003 (size=196)
2024-12-11T16:23:40,107 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-11T16:23:40,108 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-11T16:23:40,108 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-11T16:23:40,111 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39925 is added to blk_1073741827_1003 (size=196)
2024-12-11T16:23:40,136 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46841 is added to blk_1073741828_1004 (size=1189)
2024-12-11T16:23:40,136 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39925 is added to blk_1073741828_1004 (size=1189)
2024-12-11T16:23:40,138 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:34957/user/jenkins/test-data/5bdccf7f-28dc-94ce-526d-9d4d9920011a/MasterData/data/master/store
2024-12-11T16:23:40,169 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46841 is added to blk_1073741829_1005 (size=34)
2024-12-11T16:23:40,169 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39925 is added to blk_1073741829_1005 (size=34)
2024-12-11T16:23:40,171 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T16:23:40,171 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-11T16:23:40,171 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T16:23:40,171 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T16:23:40,171 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-11T16:23:40,171 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T16:23:40,171 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T16:23:40,172 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733934220171Disabling compacts and flushes for region at 1733934220171Disabling writes for close at 1733934220171Writing region close event to WAL at 1733934220171Closed at 1733934220171
2024-12-11T16:23:40,179 WARN  [master/7446a65196a1:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:34957/user/jenkins/test-data/5bdccf7f-28dc-94ce-526d-9d4d9920011a/MasterData/data/master/store/.initializing
2024-12-11T16:23:40,179 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:34957/user/jenkins/test-data/5bdccf7f-28dc-94ce-526d-9d4d9920011a/MasterData/WALs/7446a65196a1,35575,1733934219451
2024-12-11T16:23:40,191 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7446a65196a1%2C35575%2C1733934219451, suffix=, logDir=hdfs://localhost:34957/user/jenkins/test-data/5bdccf7f-28dc-94ce-526d-9d4d9920011a/MasterData/WALs/7446a65196a1,35575,1733934219451, archiveDir=hdfs://localhost:34957/user/jenkins/test-data/5bdccf7f-28dc-94ce-526d-9d4d9920011a/MasterData/oldWALs, maxLogs=10
2024-12-11T16:23:40,192 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7446a65196a1%2C35575%2C1733934219451.1733934220192
2024-12-11T16:23:40,223 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/5bdccf7f-28dc-94ce-526d-9d4d9920011a/MasterData/WALs/7446a65196a1,35575,1733934219451/7446a65196a1%2C35575%2C1733934219451.1733934220192
2024-12-11T16:23:40,234 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44977:44977),(127.0.0.1/127.0.0.1:38513:38513)]
2024-12-11T16:23:40,243 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-11T16:23:40,244 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T16:23:40,244 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T16:23:40,244 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T16:23:40,256 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-11T16:23:40,261 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-11T16:23:40,261 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:23:40,265 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-11T16:23:40,267 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T16:23:40,267 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-11T16:23:40,274 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:23:40,280 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-11T16:23:40,280 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:23:40,282 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-11T16:23:40,282 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-11T16:23:40,288 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-11T16:23:40,288 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:23:40,289 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-11T16:23:40,290 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-11T16:23:40,297 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-11T16:23:40,298 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:23:40,305 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-11T16:23:40,305 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T16:23:40,307 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:23:40,307 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34957/user/jenkins/test-data/5bdccf7f-28dc-94ce-526d-9d4d9920011a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-11T16:23:40,307 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34957/user/jenkins/test-data/5bdccf7f-28dc-94ce-526d-9d4d9920011a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-11T16:23:40,313 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T16:23:40,313 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T16:23:40,314 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-11T16:23:40,318 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T16:23:40,335 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34957/user/jenkins/test-data/5bdccf7f-28dc-94ce-526d-9d4d9920011a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-11T16:23:40,337 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=749399, jitterRate=-0.0470903217792511}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-11T16:23:40,339 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1733934220244Initializing all the Stores at 1733934220255 (+11 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733934220255Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733934220256 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733934220256Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733934220256Cleaning up temporary data from old regions at 1733934220313 (+57 ms)Region opened successfully at 1733934220339 (+26 ms)
2024-12-11T16:23:40,343 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-11T16:23:40,366 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65680bfb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7446a65196a1/172.17.0.2:0
2024-12-11T16:23:40,373 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-12-11T16:23:40,373 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-11T16:23:40,374 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-11T16:23:40,374 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-11T16:23:40,376 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 2 msec
2024-12-11T16:23:40,377 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-12-11T16:23:40,377 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-11T16:23:40,404 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-11T16:23:40,411 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35575-0x10015be3d850000, quorum=127.0.0.1:50343, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-11T16:23:40,439 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-12-11T16:23:40,440 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-11T16:23:40,443 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35575-0x10015be3d850000, quorum=127.0.0.1:50343, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-11T16:23:40,466 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-12-11T16:23:40,467 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-11T16:23:40,479 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35575-0x10015be3d850000, quorum=127.0.0.1:50343, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-11T16:23:40,503 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-12-11T16:23:40,510 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35575-0x10015be3d850000, quorum=127.0.0.1:50343, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-11T16:23:40,533 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-12-11T16:23:40,540 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35575-0x10015be3d850000, quorum=127.0.0.1:50343, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-11T16:23:40,553 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-11T16:23:40,561 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41351-0x10015be3d850001, quorum=127.0.0.1:50343, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-11T16:23:40,561 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35575-0x10015be3d850000, quorum=127.0.0.1:50343, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-11T16:23:40,561 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41351-0x10015be3d850001, quorum=127.0.0.1:50343, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:23:40,561 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35575-0x10015be3d850000, quorum=127.0.0.1:50343, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:23:40,563 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=7446a65196a1,35575,1733934219451, sessionid=0x10015be3d850000, setting cluster-up flag (Was=false)
2024-12-11T16:23:40,594 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41351-0x10015be3d850001, quorum=127.0.0.1:50343, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:23:40,594 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35575-0x10015be3d850000, quorum=127.0.0.1:50343, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:23:40,619 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-11T16:23:40,621 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=7446a65196a1,35575,1733934219451
2024-12-11T16:23:40,644 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35575-0x10015be3d850000, quorum=127.0.0.1:50343, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:23:40,644 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41351-0x10015be3d850001, quorum=127.0.0.1:50343, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:23:40,678 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-11T16:23:40,685 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=7446a65196a1,35575,1733934219451
2024-12-11T16:23:40,687 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:34957/user/jenkins/test-data/5bdccf7f-28dc-94ce-526d-9d4d9920011a/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-12-11T16:23:40,692 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-12-11T16:23:40,693 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-12-11T16:23:40,693 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-11T16:23:40,693 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 7446a65196a1,35575,1733934219451
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-11T16:23:40,697 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/7446a65196a1:0, corePoolSize=5, maxPoolSize=5
2024-12-11T16:23:40,697 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/7446a65196a1:0, corePoolSize=5, maxPoolSize=5
2024-12-11T16:23:40,697 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/7446a65196a1:0, corePoolSize=5, maxPoolSize=5
2024-12-11T16:23:40,697 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/7446a65196a1:0, corePoolSize=5, maxPoolSize=5
2024-12-11T16:23:40,697 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/7446a65196a1:0, corePoolSize=10, maxPoolSize=10
2024-12-11T16:23:40,697 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:23:40,697 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/7446a65196a1:0, corePoolSize=2, maxPoolSize=2
2024-12-11T16:23:40,697 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:23:40,701 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-11T16:23:40,701 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-12-11T16:23:40,703 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:23:40,703 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733934250703
2024-12-11T16:23:40,703 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-11T16:23:40,703 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-11T16:23:40,703 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-11T16:23:40,703 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-11T16:23:40,703 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-11T16:23:40,703 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-11T16:23:40,704 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-11T16:23:40,704 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-11T16:23:40,704 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-11T16:23:40,705 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-11T16:23:40,705 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-11T16:23:40,706 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-11T16:23:40,706 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-11T16:23:40,707 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/7446a65196a1:0:becomeActiveMaster-HFileCleaner.large.0-1733934220706,5,FailOnTimeoutGroup]
2024-12-11T16:23:40,707 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/7446a65196a1:0:becomeActiveMaster-HFileCleaner.small.0-1733934220707,5,FailOnTimeoutGroup]
2024-12-11T16:23:40,707 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-11T16:23:40,707 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-11T16:23:40,708 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-11T16:23:40,708 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-11T16:23:40,721 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39925 is added to blk_1073741831_1007 (size=1321)
2024-12-11T16:23:40,721 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46841 is added to blk_1073741831_1007 (size=1321)
2024-12-11T16:23:40,732 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:34957/user/jenkins/test-data/5bdccf7f-28dc-94ce-526d-9d4d9920011a/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-12-11T16:23:40,732 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:34957/user/jenkins/test-data/5bdccf7f-28dc-94ce-526d-9d4d9920011a
2024-12-11T16:23:40,742 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39925 is added to blk_1073741832_1008 (size=32)
2024-12-11T16:23:40,743 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T16:23:40,745 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-11T16:23:40,749 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46841 is added to blk_1073741832_1008 (size=32)
2024-12-11T16:23:40,749 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-11T16:23:40,749 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:23:40,750 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T16:23:40,750 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-11T16:23:40,753 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-11T16:23:40,753 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:23:40,754 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T16:23:40,754 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-11T16:23:40,759 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-11T16:23:40,760 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:23:40,760 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T16:23:40,761 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-11T16:23:40,763 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-11T16:23:40,763 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:23:40,766 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T16:23:40,766 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-11T16:23:40,767 INFO  [RS:0;7446a65196a1:41351 {}] regionserver.HRegionServer(746): ClusterId : c5ba6bf3-5a92-4957-b073-9acdde23983a
2024-12-11T16:23:40,767 DEBUG [RS:0;7446a65196a1:41351 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-11T16:23:40,769 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34957/user/jenkins/test-data/5bdccf7f-28dc-94ce-526d-9d4d9920011a/data/hbase/meta/1588230740
2024-12-11T16:23:40,773 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34957/user/jenkins/test-data/5bdccf7f-28dc-94ce-526d-9d4d9920011a/data/hbase/meta/1588230740
2024-12-11T16:23:40,775 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-11T16:23:40,775 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-11T16:23:40,776 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-11T16:23:40,777 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-11T16:23:40,780 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34957/user/jenkins/test-data/5bdccf7f-28dc-94ce-526d-9d4d9920011a/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-11T16:23:40,780 DEBUG [RS:0;7446a65196a1:41351 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-11T16:23:40,780 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=856492, jitterRate=0.08908633887767792}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-11T16:23:40,781 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1733934220744Initializing all the Stores at 1733934220745 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733934220745Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733934220745Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733934220745Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733934220745Cleaning up temporary data from old regions at 1733934220775 (+30 ms)Region opened successfully at 1733934220781 (+6 ms)
2024-12-11T16:23:40,781 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-11T16:23:40,781 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-11T16:23:40,781 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-11T16:23:40,781 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-11T16:23:40,781 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-11T16:23:40,782 DEBUG [RS:0;7446a65196a1:41351 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-11T16:23:40,782 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-11T16:23:40,782 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733934220781Disabling compacts and flushes for region at 1733934220781Disabling writes for close at 1733934220781Writing region close event to WAL at 1733934220782 (+1 ms)Closed at 1733934220782
2024-12-11T16:23:40,785 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-11T16:23:40,785 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-12-11T16:23:40,785 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-11T16:23:40,788 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-11T16:23:40,791 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-11T16:23:40,798 DEBUG [RS:0;7446a65196a1:41351 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-11T16:23:40,798 DEBUG [RS:0;7446a65196a1:41351 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6355dff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7446a65196a1/172.17.0.2:0
2024-12-11T16:23:40,812 DEBUG [RS:0;7446a65196a1:41351 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;7446a65196a1:41351
2024-12-11T16:23:40,812 INFO  [RS:0;7446a65196a1:41351 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-12-11T16:23:40,812 INFO  [RS:0;7446a65196a1:41351 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-12-11T16:23:40,812 DEBUG [RS:0;7446a65196a1:41351 {}] regionserver.HRegionServer(832): About to register with Master.
2024-12-11T16:23:40,813 INFO  [RS:0;7446a65196a1:41351 {}] regionserver.HRegionServer(2659): reportForDuty to master=7446a65196a1,35575,1733934219451 with port=41351, startcode=1733934219643
2024-12-11T16:23:40,814 DEBUG [RS:0;7446a65196a1:41351 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-11T16:23:40,833 INFO  [HMaster-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33531, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService
2024-12-11T16:23:40,834 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35575 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 7446a65196a1,41351,1733934219643
2024-12-11T16:23:40,834 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35575 {}] master.ServerManager(517): Registering regionserver=7446a65196a1,41351,1733934219643
2024-12-11T16:23:40,837 DEBUG [RS:0;7446a65196a1:41351 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:34957/user/jenkins/test-data/5bdccf7f-28dc-94ce-526d-9d4d9920011a
2024-12-11T16:23:40,837 DEBUG [RS:0;7446a65196a1:41351 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:34957
2024-12-11T16:23:40,837 DEBUG [RS:0;7446a65196a1:41351 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-12-11T16:23:40,845 DEBUG [RS:0;7446a65196a1:41351 {}] zookeeper.ZKUtil(111): regionserver:41351-0x10015be3d850001, quorum=127.0.0.1:50343, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/7446a65196a1,41351,1733934219643
2024-12-11T16:23:40,845 WARN  [RS:0;7446a65196a1:41351 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-11T16:23:40,845 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35575-0x10015be3d850000, quorum=127.0.0.1:50343, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-11T16:23:40,845 INFO  [RS:0;7446a65196a1:41351 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-11T16:23:40,845 DEBUG [RS:0;7446a65196a1:41351 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:34957/user/jenkins/test-data/5bdccf7f-28dc-94ce-526d-9d4d9920011a/WALs/7446a65196a1,41351,1733934219643
2024-12-11T16:23:40,847 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [7446a65196a1,41351,1733934219643]
2024-12-11T16:23:40,863 INFO  [RS:0;7446a65196a1:41351 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-11T16:23:40,866 INFO  [RS:0;7446a65196a1:41351 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-11T16:23:40,866 INFO  [RS:0;7446a65196a1:41351 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-11T16:23:40,866 INFO  [RS:0;7446a65196a1:41351 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-11T16:23:40,871 INFO  [RS:0;7446a65196a1:41351 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-12-11T16:23:40,872 INFO  [RS:0;7446a65196a1:41351 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-12-11T16:23:40,872 INFO  [RS:0;7446a65196a1:41351 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-11T16:23:40,872 DEBUG [RS:0;7446a65196a1:41351 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:23:40,872 DEBUG [RS:0;7446a65196a1:41351 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:23:40,872 DEBUG [RS:0;7446a65196a1:41351 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:23:40,872 DEBUG [RS:0;7446a65196a1:41351 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:23:40,872 DEBUG [RS:0;7446a65196a1:41351 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:23:40,872 DEBUG [RS:0;7446a65196a1:41351 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/7446a65196a1:0, corePoolSize=2, maxPoolSize=2
2024-12-11T16:23:40,873 DEBUG [RS:0;7446a65196a1:41351 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:23:40,873 DEBUG [RS:0;7446a65196a1:41351 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:23:40,873 DEBUG [RS:0;7446a65196a1:41351 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:23:40,873 DEBUG [RS:0;7446a65196a1:41351 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:23:40,873 DEBUG [RS:0;7446a65196a1:41351 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:23:40,873 DEBUG [RS:0;7446a65196a1:41351 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:23:40,873 DEBUG [RS:0;7446a65196a1:41351 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/7446a65196a1:0, corePoolSize=3, maxPoolSize=3
2024-12-11T16:23:40,873 DEBUG [RS:0;7446a65196a1:41351 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/7446a65196a1:0, corePoolSize=3, maxPoolSize=3
2024-12-11T16:23:40,875 INFO  [RS:0;7446a65196a1:41351 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-11T16:23:40,875 INFO  [RS:0;7446a65196a1:41351 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-11T16:23:40,875 INFO  [RS:0;7446a65196a1:41351 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-11T16:23:40,875 INFO  [RS:0;7446a65196a1:41351 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-11T16:23:40,875 INFO  [RS:0;7446a65196a1:41351 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-11T16:23:40,875 INFO  [RS:0;7446a65196a1:41351 {}] hbase.ChoreService(168): Chore ScheduledChore name=7446a65196a1,41351,1733934219643-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-11T16:23:40,904 INFO  [RS:0;7446a65196a1:41351 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-11T16:23:40,904 INFO  [RS:0;7446a65196a1:41351 {}] hbase.ChoreService(168): Chore ScheduledChore name=7446a65196a1,41351,1733934219643-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-11T16:23:40,904 INFO  [RS:0;7446a65196a1:41351 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T16:23:40,905 INFO  [RS:0;7446a65196a1:41351 {}] regionserver.Replication(171): 7446a65196a1,41351,1733934219643 started
2024-12-11T16:23:40,927 INFO  [RS:0;7446a65196a1:41351 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T16:23:40,928 INFO  [RS:0;7446a65196a1:41351 {}] regionserver.HRegionServer(1482): Serving as 7446a65196a1,41351,1733934219643, RpcServer on 7446a65196a1/172.17.0.2:41351, sessionid=0x10015be3d850001
2024-12-11T16:23:40,928 DEBUG [RS:0;7446a65196a1:41351 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-11T16:23:40,928 DEBUG [RS:0;7446a65196a1:41351 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 7446a65196a1,41351,1733934219643
2024-12-11T16:23:40,928 DEBUG [RS:0;7446a65196a1:41351 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7446a65196a1,41351,1733934219643'
2024-12-11T16:23:40,928 DEBUG [RS:0;7446a65196a1:41351 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-11T16:23:40,935 DEBUG [RS:0;7446a65196a1:41351 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-11T16:23:40,939 DEBUG [RS:0;7446a65196a1:41351 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-11T16:23:40,939 DEBUG [RS:0;7446a65196a1:41351 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-11T16:23:40,940 DEBUG [RS:0;7446a65196a1:41351 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 7446a65196a1,41351,1733934219643
2024-12-11T16:23:40,940 DEBUG [RS:0;7446a65196a1:41351 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7446a65196a1,41351,1733934219643'
2024-12-11T16:23:40,940 DEBUG [RS:0;7446a65196a1:41351 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-11T16:23:40,941 DEBUG [RS:0;7446a65196a1:41351 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-11T16:23:40,942 DEBUG [RS:0;7446a65196a1:41351 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-11T16:23:40,942 INFO  [RS:0;7446a65196a1:41351 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-11T16:23:40,942 INFO  [RS:0;7446a65196a1:41351 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-11T16:23:40,945 WARN  [7446a65196a1:35575 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-12-11T16:23:41,047 INFO  [RS:0;7446a65196a1:41351 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7446a65196a1%2C41351%2C1733934219643, suffix=, logDir=hdfs://localhost:34957/user/jenkins/test-data/5bdccf7f-28dc-94ce-526d-9d4d9920011a/WALs/7446a65196a1,41351,1733934219643, archiveDir=hdfs://localhost:34957/user/jenkins/test-data/5bdccf7f-28dc-94ce-526d-9d4d9920011a/oldWALs, maxLogs=32
2024-12-11T16:23:41,049 INFO  [RS:0;7446a65196a1:41351 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7446a65196a1%2C41351%2C1733934219643.1733934221049
2024-12-11T16:23:41,140 INFO  [RS:0;7446a65196a1:41351 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/5bdccf7f-28dc-94ce-526d-9d4d9920011a/WALs/7446a65196a1,41351,1733934219643/7446a65196a1%2C41351%2C1733934219643.1733934221049
2024-12-11T16:23:41,165 DEBUG [RS:0;7446a65196a1:41351 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44977:44977),(127.0.0.1/127.0.0.1:38513:38513)]
2024-12-11T16:23:41,195 DEBUG [7446a65196a1:35575 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-12-11T16:23:41,199 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=7446a65196a1,41351,1733934219643
2024-12-11T16:23:41,220 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7446a65196a1,41351,1733934219643, state=OPENING
2024-12-11T16:23:41,241 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-11T16:23:41,259 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41351-0x10015be3d850001, quorum=127.0.0.1:50343, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:23:41,259 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35575-0x10015be3d850000, quorum=127.0.0.1:50343, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:23:41,260 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-11T16:23:41,261 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-11T16:23:41,261 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-11T16:23:41,261 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=7446a65196a1,41351,1733934219643}]
2024-12-11T16:23:41,418 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-11T16:23:41,439 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52323, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-11T16:23:41,448 INFO  [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-12-11T16:23:41,449 INFO  [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-11T16:23:41,458 INFO  [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7446a65196a1%2C41351%2C1733934219643.meta, suffix=.meta, logDir=hdfs://localhost:34957/user/jenkins/test-data/5bdccf7f-28dc-94ce-526d-9d4d9920011a/WALs/7446a65196a1,41351,1733934219643, archiveDir=hdfs://localhost:34957/user/jenkins/test-data/5bdccf7f-28dc-94ce-526d-9d4d9920011a/oldWALs, maxLogs=32
2024-12-11T16:23:41,463 INFO  [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 7446a65196a1%2C41351%2C1733934219643.meta.1733934221463.meta
2024-12-11T16:23:41,529 INFO  [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/5bdccf7f-28dc-94ce-526d-9d4d9920011a/WALs/7446a65196a1,41351,1733934219643/7446a65196a1%2C41351%2C1733934219643.meta.1733934221463.meta
2024-12-11T16:23:41,559 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44977:44977),(127.0.0.1/127.0.0.1:38513:38513)]
2024-12-11T16:23:41,579 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-11T16:23:41,580 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-11T16:23:41,580 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-11T16:23:41,580 INFO  [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-11T16:23:41,580 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-11T16:23:41,581 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T16:23:41,581 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-12-11T16:23:41,581 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-12-11T16:23:41,599 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-11T16:23:41,601 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-11T16:23:41,601 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:23:41,604 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T16:23:41,604 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-11T16:23:41,608 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-11T16:23:41,608 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:23:41,612 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T16:23:41,612 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-11T16:23:41,614 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-11T16:23:41,614 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:23:41,620 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T16:23:41,620 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-11T16:23:41,623 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-11T16:23:41,623 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:23:41,631 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T16:23:41,632 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-11T16:23:41,635 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34957/user/jenkins/test-data/5bdccf7f-28dc-94ce-526d-9d4d9920011a/data/hbase/meta/1588230740
2024-12-11T16:23:41,647 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34957/user/jenkins/test-data/5bdccf7f-28dc-94ce-526d-9d4d9920011a/data/hbase/meta/1588230740
2024-12-11T16:23:41,651 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-11T16:23:41,652 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-11T16:23:41,653 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-11T16:23:41,656 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-11T16:23:41,657 INFO  [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=755791, jitterRate=-0.03896220028400421}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-11T16:23:41,657 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-12-11T16:23:41,658 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1733934221581Writing region info on filesystem at 1733934221581Initializing all the Stores at 1733934221592 (+11 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733934221592Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733934221595 (+3 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733934221595Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733934221595Cleaning up temporary data from old regions at 1733934221652 (+57 ms)Running coprocessor post-open hooks at 1733934221657 (+5 ms)Region opened successfully at 1733934221658 (+1 ms)
2024-12-11T16:23:41,659 INFO  [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733934221418
2024-12-11T16:23:41,663 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-11T16:23:41,663 INFO  [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-12-11T16:23:41,664 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=7446a65196a1,41351,1733934219643
2024-12-11T16:23:41,666 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7446a65196a1,41351,1733934219643, state=OPEN
2024-12-11T16:23:41,713 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41351-0x10015be3d850001, quorum=127.0.0.1:50343, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-11T16:23:41,715 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-11T16:23:41,715 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35575-0x10015be3d850000, quorum=127.0.0.1:50343, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-11T16:23:41,716 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-11T16:23:41,716 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=7446a65196a1,41351,1733934219643
2024-12-11T16:23:41,745 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-12-11T16:23:41,745 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7446a65196a1,41351,1733934219643 in 455 msec
2024-12-11T16:23:41,769 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-12-11T16:23:41,769 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 961 msec
2024-12-11T16:23:41,775 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-11T16:23:41,775 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-12-11T16:23:41,777 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-11T16:23:41,777 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7446a65196a1,41351,1733934219643, seqNum=-1]
2024-12-11T16:23:41,778 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-11T16:23:41,780 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32801, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-11T16:23:41,800 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1010 sec
2024-12-11T16:23:41,800 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733934221800, completionTime=-1
2024-12-11T16:23:41,800 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-12-11T16:23:41,800 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-12-11T16:23:41,805 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-12-11T16:23:41,805 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733934281805
2024-12-11T16:23:41,805 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733934341805
2024-12-11T16:23:41,805 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 4 msec
2024-12-11T16:23:41,806 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7446a65196a1,35575,1733934219451-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-11T16:23:41,806 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7446a65196a1,35575,1733934219451-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T16:23:41,806 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7446a65196a1,35575,1733934219451-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T16:23:41,806 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-7446a65196a1:35575, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T16:23:41,806 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-11T16:23:41,807 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-12-11T16:23:41,813 DEBUG [master/7446a65196a1:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-12-11T16:23:41,815 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.973sec
2024-12-11T16:23:41,815 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-11T16:23:41,815 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-11T16:23:41,815 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-11T16:23:41,815 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-11T16:23:41,815 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-11T16:23:41,815 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7446a65196a1,35575,1733934219451-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-11T16:23:41,816 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7446a65196a1,35575,1733934219451-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-11T16:23:41,845 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-12-11T16:23:41,846 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-11T16:23:41,846 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7446a65196a1,35575,1733934219451-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T16:23:41,867 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69958721, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-11T16:23:41,867 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7446a65196a1,35575,-1 for getting cluster id
2024-12-11T16:23:41,868 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-12-11T16:23:41,878 DEBUG [HMaster-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c5ba6bf3-5a92-4957-b073-9acdde23983a'
2024-12-11T16:23:41,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-12-11T16:23:41,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c5ba6bf3-5a92-4957-b073-9acdde23983a"

2024-12-11T16:23:41,880 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b52694f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-11T16:23:41,880 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7446a65196a1,35575,-1]
2024-12-11T16:23:41,880 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-12-11T16:23:41,881 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T16:23:41,883 INFO  [HMaster-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55490, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-12-11T16:23:41,884 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bad3a06, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-11T16:23:41,884 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-11T16:23:41,885 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7446a65196a1,41351,1733934219643, seqNum=-1]
2024-12-11T16:23:41,886 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-11T16:23:41,891 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40148, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-11T16:23:41,896 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=7446a65196a1,35575,1733934219451
2024-12-11T16:23:41,897 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T16:23:41,902 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-12-11T16:23:41,903 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-12-11T16:23:41,903 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-11T16:23:41,903 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at org.apache.hadoop.hbase.regionserver.wal.TestLogRolling.testLogRollOnDatanodeDeath(TestLogRolling.java:201)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-11T16:23:41,903 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T16:23:41,903 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T16:23:41,903 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-12-11T16:23:41,904 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-12-11T16:23:41,904 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=69112748, stopped=false
2024-12-11T16:23:41,904 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=7446a65196a1,35575,1733934219451
2024-12-11T16:23:41,927 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41351-0x10015be3d850001, quorum=127.0.0.1:50343, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-11T16:23:41,927 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41351-0x10015be3d850001, quorum=127.0.0.1:50343, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:23:41,928 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-11T16:23:41,929 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-11T16:23:41,929 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at org.apache.hadoop.hbase.regionserver.wal.TestLogRolling.testLogRollOnDatanodeDeath(TestLogRolling.java:201)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-11T16:23:41,929 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T16:23:41,930 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '7446a65196a1,41351,1733934219643' *****
2024-12-11T16:23:41,930 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-12-11T16:23:41,931 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35575-0x10015be3d850000, quorum=127.0.0.1:50343, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-11T16:23:41,931 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35575-0x10015be3d850000, quorum=127.0.0.1:50343, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:23:41,935 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35575-0x10015be3d850000, quorum=127.0.0.1:50343, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-11T16:23:41,938 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41351-0x10015be3d850001, quorum=127.0.0.1:50343, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-11T16:23:41,939 INFO  [RS:0;7446a65196a1:41351 {}] regionserver.HRegionServer(878): Closing user regions
2024-12-11T16:23:41,948 INFO  [RS:0;7446a65196a1:41351 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-11T16:23:41,948 INFO  [RS:0;7446a65196a1:41351 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-11T16:23:41,948 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-12-11T16:23:41,948 INFO  [RS:0;7446a65196a1:41351 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-11T16:23:41,948 INFO  [RS:0;7446a65196a1:41351 {}] regionserver.HRegionServer(959): stopping server 7446a65196a1,41351,1733934219643
2024-12-11T16:23:41,948 INFO  [RS:0;7446a65196a1:41351 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-11T16:23:41,948 INFO  [RS:0;7446a65196a1:41351 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;7446a65196a1:41351.
2024-12-11T16:23:41,948 DEBUG [RS:0;7446a65196a1:41351 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-11T16:23:41,948 DEBUG [RS:0;7446a65196a1:41351 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T16:23:41,949 INFO  [RS:0;7446a65196a1:41351 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-11T16:23:41,951 INFO  [RS:0;7446a65196a1:41351 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-11T16:23:41,951 INFO  [RS:0;7446a65196a1:41351 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-11T16:23:41,952 INFO  [RS:0;7446a65196a1:41351 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-12-11T16:23:41,957 INFO  [RS:0;7446a65196a1:41351 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close
2024-12-11T16:23:41,957 DEBUG [RS:0;7446a65196a1:41351 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740}
2024-12-11T16:23:41,957 DEBUG [RS:0;7446a65196a1:41351 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-12-11T16:23:41,957 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-11T16:23:41,958 INFO  [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-11T16:23:41,958 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-11T16:23:41,958 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-11T16:23:41,958 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-11T16:23:41,958 INFO  [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=74 B heapSize=1.22 KB
2024-12-11T16:23:41,992 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34957/user/jenkins/test-data/5bdccf7f-28dc-94ce-526d-9d4d9920011a/data/hbase/meta/1588230740/.tmp/ns/8ad95bfcec0249b09a76d1942a5b82b2 is 43, key is default/ns:d/1733934221781/Put/seqid=0
2024-12-11T16:23:42,005 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46841 is added to blk_1073741835_1011 (size=5153)
2024-12-11T16:23:42,005 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39925 is added to blk_1073741835_1011 (size=5153)
2024-12-11T16:23:42,006 INFO  [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:34957/user/jenkins/test-data/5bdccf7f-28dc-94ce-526d-9d4d9920011a/data/hbase/meta/1588230740/.tmp/ns/8ad95bfcec0249b09a76d1942a5b82b2
2024-12-11T16:23:42,016 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34957/user/jenkins/test-data/5bdccf7f-28dc-94ce-526d-9d4d9920011a/data/hbase/meta/1588230740/.tmp/ns/8ad95bfcec0249b09a76d1942a5b82b2 as hdfs://localhost:34957/user/jenkins/test-data/5bdccf7f-28dc-94ce-526d-9d4d9920011a/data/hbase/meta/1588230740/ns/8ad95bfcec0249b09a76d1942a5b82b2
2024-12-11T16:23:42,023 INFO  [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34957/user/jenkins/test-data/5bdccf7f-28dc-94ce-526d-9d4d9920011a/data/hbase/meta/1588230740/ns/8ad95bfcec0249b09a76d1942a5b82b2, entries=2, sequenceid=6, filesize=5.0 K
2024-12-11T16:23:42,025 INFO  [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 67ms, sequenceid=6, compaction requested=false
2024-12-11T16:23:42,025 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-12-11T16:23:42,032 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34957/user/jenkins/test-data/5bdccf7f-28dc-94ce-526d-9d4d9920011a/data/hbase/meta/1588230740/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1
2024-12-11T16:23:42,033 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-11T16:23:42,033 INFO  [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-11T16:23:42,033 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733934221957Running coprocessor pre-close hooks at 1733934221957Disabling compacts and flushes for region at 1733934221957Disabling writes for close at 1733934221958 (+1 ms)Obtaining lock to block concurrent updates at 1733934221958Preparing flush snapshotting stores in 1588230740 at 1733934221958Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=74, getHeapSize=1184, getOffHeapSize=0, getCellsCount=2 at 1733934221959 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733934221963 (+4 ms)Flushing 1588230740/ns: creating writer at 1733934221963Flushing 1588230740/ns: appending metadata at 1733934221991 (+28 ms)Flushing 1588230740/ns: closing flushed file at 1733934221991Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2ab2586a: reopening flushed file at 1733934222015 (+24 ms)Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 67ms, sequenceid=6, compaction requested=false at 1733934222025 (+10 ms)Writing region close event to WAL at 1733934222027 (+2 ms)Running coprocessor post-close hooks at 1733934222033 (+6 ms)Closed at 1733934222033
2024-12-11T16:23:42,033 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-12-11T16:23:42,158 INFO  [RS:0;7446a65196a1:41351 {}] regionserver.HRegionServer(976): stopping server 7446a65196a1,41351,1733934219643; all regions closed.
2024-12-11T16:23:42,158 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:23:42,159 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:23:42,159 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:23:42,159 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:23:42,159 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:23:42,179 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39925 is added to blk_1073741834_1010 (size=1152)
2024-12-11T16:23:42,188 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46841 is added to blk_1073741834_1010 (size=1152)
2024-12-11T16:23:42,188 DEBUG [RS:0;7446a65196a1:41351 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/5bdccf7f-28dc-94ce-526d-9d4d9920011a/oldWALs
2024-12-11T16:23:42,188 INFO  [RS:0;7446a65196a1:41351 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 7446a65196a1%2C41351%2C1733934219643.meta:.meta(num 1733934221463)
2024-12-11T16:23:42,189 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:23:42,189 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:23:42,190 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:23:42,190 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:23:42,194 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:23:42,211 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39925 is added to blk_1073741833_1009 (size=93)
2024-12-11T16:23:42,222 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46841 is added to blk_1073741833_1009 (size=93)
2024-12-11T16:23:42,247 DEBUG [RS:0;7446a65196a1:41351 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/5bdccf7f-28dc-94ce-526d-9d4d9920011a/oldWALs
2024-12-11T16:23:42,247 INFO  [RS:0;7446a65196a1:41351 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 7446a65196a1%2C41351%2C1733934219643:(num 1733934221049)
2024-12-11T16:23:42,247 DEBUG [RS:0;7446a65196a1:41351 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T16:23:42,247 INFO  [RS:0;7446a65196a1:41351 {}] regionserver.LeaseManager(133): Closed leases
2024-12-11T16:23:42,247 INFO  [RS:0;7446a65196a1:41351 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-11T16:23:42,248 INFO  [RS:0;7446a65196a1:41351 {}] hbase.ChoreService(370): Chore service for: regionserver/7446a65196a1:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown
2024-12-11T16:23:42,248 INFO  [RS:0;7446a65196a1:41351 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-11T16:23:42,248 INFO  [RS:0;7446a65196a1:41351 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41351
2024-12-11T16:23:42,248 INFO  [regionserver/7446a65196a1:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-11T16:23:42,269 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41351-0x10015be3d850001, quorum=127.0.0.1:50343, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/7446a65196a1,41351,1733934219643
2024-12-11T16:23:42,275 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35575-0x10015be3d850000, quorum=127.0.0.1:50343, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-11T16:23:42,279 INFO  [RS:0;7446a65196a1:41351 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-11T16:23:42,299 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [7446a65196a1,41351,1733934219643]
2024-12-11T16:23:42,319 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/7446a65196a1,41351,1733934219643 already deleted, retry=false
2024-12-11T16:23:42,319 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 7446a65196a1,41351,1733934219643 expired; onlineServers=0
2024-12-11T16:23:42,319 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '7446a65196a1,35575,1733934219451' *****
2024-12-11T16:23:42,319 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-12-11T16:23:42,323 INFO  [M:0;7446a65196a1:35575 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-11T16:23:42,323 INFO  [M:0;7446a65196a1:35575 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-11T16:23:42,323 DEBUG [M:0;7446a65196a1:35575 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-12-11T16:23:42,323 DEBUG [M:0;7446a65196a1:35575 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-12-11T16:23:42,323 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-12-11T16:23:42,323 DEBUG [master/7446a65196a1:0:becomeActiveMaster-HFileCleaner.small.0-1733934220707 {}] cleaner.HFileCleaner(306): Exit Thread[master/7446a65196a1:0:becomeActiveMaster-HFileCleaner.small.0-1733934220707,5,FailOnTimeoutGroup]
2024-12-11T16:23:42,323 INFO  [M:0;7446a65196a1:35575 {}] hbase.ChoreService(370): Chore service for: master/7446a65196a1:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-12-11T16:23:42,324 INFO  [M:0;7446a65196a1:35575 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-11T16:23:42,324 DEBUG [M:0;7446a65196a1:35575 {}] master.HMaster(1795): Stopping service threads
2024-12-11T16:23:42,324 INFO  [M:0;7446a65196a1:35575 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-12-11T16:23:42,324 INFO  [M:0;7446a65196a1:35575 {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-11T16:23:42,324 INFO  [M:0;7446a65196a1:35575 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-12-11T16:23:42,325 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-12-11T16:23:42,327 DEBUG [master/7446a65196a1:0:becomeActiveMaster-HFileCleaner.large.0-1733934220706 {}] cleaner.HFileCleaner(306): Exit Thread[master/7446a65196a1:0:becomeActiveMaster-HFileCleaner.large.0-1733934220706,5,FailOnTimeoutGroup]
2024-12-11T16:23:42,341 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35575-0x10015be3d850000, quorum=127.0.0.1:50343, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-12-11T16:23:42,341 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35575-0x10015be3d850000, quorum=127.0.0.1:50343, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:23:42,353 DEBUG [M:0;7446a65196a1:35575 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/master already deleted, retry=false
2024-12-11T16:23:42,353 DEBUG [M:0;7446a65196a1:35575 {}] master.ActiveMasterManager(353): master:35575-0x10015be3d850000, quorum=127.0.0.1:50343, baseZNode=/hbase Failed delete of our master address node; KeeperErrorCode = NoNode for /hbase/master
2024-12-11T16:23:42,367 INFO  [M:0;7446a65196a1:35575 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:34957/user/jenkins/test-data/5bdccf7f-28dc-94ce-526d-9d4d9920011a/.lastflushedseqids
2024-12-11T16:23:42,396 INFO  [RS:0;7446a65196a1:41351 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-11T16:23:42,396 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41351-0x10015be3d850001, quorum=127.0.0.1:50343, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-11T16:23:42,396 INFO  [RS:0;7446a65196a1:41351 {}] regionserver.HRegionServer(1031): Exiting; stopping=7446a65196a1,41351,1733934219643; zookeeper connection closed.
2024-12-11T16:23:42,396 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41351-0x10015be3d850001, quorum=127.0.0.1:50343, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-11T16:23:42,415 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@39dafeac {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@39dafeac
2024-12-11T16:23:42,423 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-12-11T16:23:42,431 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46841 is added to blk_1073741836_1012 (size=99)
2024-12-11T16:23:42,431 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39925 is added to blk_1073741836_1012 (size=99)
2024-12-11T16:23:42,840 INFO  [M:0;7446a65196a1:35575 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-12-11T16:23:42,840 INFO  [M:0;7446a65196a1:35575 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-12-11T16:23:42,840 DEBUG [M:0;7446a65196a1:35575 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-11T16:23:42,841 INFO  [M:0;7446a65196a1:35575 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T16:23:42,841 DEBUG [M:0;7446a65196a1:35575 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T16:23:42,841 DEBUG [M:0;7446a65196a1:35575 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-11T16:23:42,841 DEBUG [M:0;7446a65196a1:35575 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T16:23:42,841 INFO  [M:0;7446a65196a1:35575 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=7.67 KB heapSize=11.34 KB
2024-12-11T16:23:42,883 INFO  [regionserver/7446a65196a1:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-11T16:23:42,912 DEBUG [M:0;7446a65196a1:35575 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34957/user/jenkins/test-data/5bdccf7f-28dc-94ce-526d-9d4d9920011a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/933fcb7a27214a2098b68028ee979a43 is 82, key is hbase:meta,,1/info:regioninfo/1733934221664/Put/seqid=0
2024-12-11T16:23:42,946 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39925 is added to blk_1073741837_1013 (size=5672)
2024-12-11T16:23:42,955 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46841 is added to blk_1073741837_1013 (size=5672)
2024-12-11T16:23:42,955 INFO  [M:0;7446a65196a1:35575 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:34957/user/jenkins/test-data/5bdccf7f-28dc-94ce-526d-9d4d9920011a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/933fcb7a27214a2098b68028ee979a43
2024-12-11T16:23:43,007 DEBUG [M:0;7446a65196a1:35575 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34957/user/jenkins/test-data/5bdccf7f-28dc-94ce-526d-9d4d9920011a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e6499d0dc76f410baf34ee315f3adecb is 240, key is \x00\x00\x00\x00\x00\x00\x00\x01/proc:d/1733934221795/Put/seqid=0
2024-12-11T16:23:43,049 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46841 is added to blk_1073741838_1014 (size=5275)
2024-12-11T16:23:43,051 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39925 is added to blk_1073741838_1014 (size=5275)
2024-12-11T16:23:43,054 INFO  [M:0;7446a65196a1:35575 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.06 KB at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:34957/user/jenkins/test-data/5bdccf7f-28dc-94ce-526d-9d4d9920011a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e6499d0dc76f410baf34ee315f3adecb
2024-12-11T16:23:43,146 DEBUG [M:0;7446a65196a1:35575 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34957/user/jenkins/test-data/5bdccf7f-28dc-94ce-526d-9d4d9920011a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/cf276859a6db4d9083d0b0446fa67631 is 69, key is 7446a65196a1,41351,1733934219643/rs:state/1733934220835/Put/seqid=0
2024-12-11T16:23:43,231 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46841 is added to blk_1073741839_1015 (size=5156)
2024-12-11T16:23:43,231 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39925 is added to blk_1073741839_1015 (size=5156)
2024-12-11T16:23:43,240 INFO  [M:0;7446a65196a1:35575 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:34957/user/jenkins/test-data/5bdccf7f-28dc-94ce-526d-9d4d9920011a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/cf276859a6db4d9083d0b0446fa67631
2024-12-11T16:23:43,332 DEBUG [M:0;7446a65196a1:35575 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34957/user/jenkins/test-data/5bdccf7f-28dc-94ce-526d-9d4d9920011a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/bf1d51378324429f9a0ff6a7d6517272 is 52, key is load_balancer_on/state:d/1733934221899/Put/seqid=0
2024-12-11T16:23:43,393 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39925 is added to blk_1073741840_1016 (size=5056)
2024-12-11T16:23:43,393 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46841 is added to blk_1073741840_1016 (size=5056)
2024-12-11T16:23:43,403 INFO  [M:0;7446a65196a1:35575 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:34957/user/jenkins/test-data/5bdccf7f-28dc-94ce-526d-9d4d9920011a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/bf1d51378324429f9a0ff6a7d6517272
2024-12-11T16:23:43,422 DEBUG [M:0;7446a65196a1:35575 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34957/user/jenkins/test-data/5bdccf7f-28dc-94ce-526d-9d4d9920011a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/933fcb7a27214a2098b68028ee979a43 as hdfs://localhost:34957/user/jenkins/test-data/5bdccf7f-28dc-94ce-526d-9d4d9920011a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/933fcb7a27214a2098b68028ee979a43
2024-12-11T16:23:43,435 INFO  [M:0;7446a65196a1:35575 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34957/user/jenkins/test-data/5bdccf7f-28dc-94ce-526d-9d4d9920011a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/933fcb7a27214a2098b68028ee979a43, entries=8, sequenceid=29, filesize=5.5 K
2024-12-11T16:23:43,437 DEBUG [M:0;7446a65196a1:35575 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34957/user/jenkins/test-data/5bdccf7f-28dc-94ce-526d-9d4d9920011a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e6499d0dc76f410baf34ee315f3adecb as hdfs://localhost:34957/user/jenkins/test-data/5bdccf7f-28dc-94ce-526d-9d4d9920011a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e6499d0dc76f410baf34ee315f3adecb
2024-12-11T16:23:43,449 INFO  [M:0;7446a65196a1:35575 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34957/user/jenkins/test-data/5bdccf7f-28dc-94ce-526d-9d4d9920011a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e6499d0dc76f410baf34ee315f3adecb, entries=3, sequenceid=29, filesize=5.2 K
2024-12-11T16:23:43,452 DEBUG [M:0;7446a65196a1:35575 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34957/user/jenkins/test-data/5bdccf7f-28dc-94ce-526d-9d4d9920011a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/cf276859a6db4d9083d0b0446fa67631 as hdfs://localhost:34957/user/jenkins/test-data/5bdccf7f-28dc-94ce-526d-9d4d9920011a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/cf276859a6db4d9083d0b0446fa67631
2024-12-11T16:23:43,466 INFO  [M:0;7446a65196a1:35575 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34957/user/jenkins/test-data/5bdccf7f-28dc-94ce-526d-9d4d9920011a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/cf276859a6db4d9083d0b0446fa67631, entries=1, sequenceid=29, filesize=5.0 K
2024-12-11T16:23:43,472 DEBUG [M:0;7446a65196a1:35575 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34957/user/jenkins/test-data/5bdccf7f-28dc-94ce-526d-9d4d9920011a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/bf1d51378324429f9a0ff6a7d6517272 as hdfs://localhost:34957/user/jenkins/test-data/5bdccf7f-28dc-94ce-526d-9d4d9920011a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/bf1d51378324429f9a0ff6a7d6517272
2024-12-11T16:23:43,525 INFO  [M:0;7446a65196a1:35575 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34957/user/jenkins/test-data/5bdccf7f-28dc-94ce-526d-9d4d9920011a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/bf1d51378324429f9a0ff6a7d6517272, entries=1, sequenceid=29, filesize=4.9 K
2024-12-11T16:23:43,527 INFO  [M:0;7446a65196a1:35575 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 685ms, sequenceid=29, compaction requested=false
2024-12-11T16:23:43,575 INFO  [M:0;7446a65196a1:35575 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T16:23:43,575 DEBUG [M:0;7446a65196a1:35575 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733934222840Disabling compacts and flushes for region at 1733934222840Disabling writes for close at 1733934222841 (+1 ms)Obtaining lock to block concurrent updates at 1733934222841Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733934222841Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=7850, getHeapSize=11544, getOffHeapSize=0, getCellsCount=36 at 1733934222842 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733934222851 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733934222851Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733934222911 (+60 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733934222911Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733934222976 (+65 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733934223006 (+30 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733934223006Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733934223086 (+80 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733934223146 (+60 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733934223146Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733934223265 (+119 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733934223331 (+66 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733934223331Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@46ba5079: reopening flushed file at 1733934223419 (+88 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@49af4df6: reopening flushed file at 1733934223435 (+16 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@36699822: reopening flushed file at 1733934223450 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@23472b7f: reopening flushed file at 1733934223466 (+16 ms)Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 685ms, sequenceid=29, compaction requested=false at 1733934223527 (+61 ms)Writing region close event to WAL at 1733934223575 (+48 ms)Closed at 1733934223575
2024-12-11T16:23:43,581 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:23:43,582 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:23:43,582 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:23:43,583 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:23:43,583 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:23:43,596 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39925 is added to blk_1073741830_1006 (size=10311)
2024-12-11T16:23:43,606 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46841 is added to blk_1073741830_1006 (size=10311)
2024-12-11T16:23:44,010 INFO  [M:0;7446a65196a1:35575 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-12-11T16:23:44,010 INFO  [M:0;7446a65196a1:35575 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35575
2024-12-11T16:23:44,010 INFO  [M:0;7446a65196a1:35575 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-11T16:23:44,011 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-11T16:23:44,147 INFO  [M:0;7446a65196a1:35575 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-11T16:23:44,147 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35575-0x10015be3d850000, quorum=127.0.0.1:50343, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-11T16:23:44,147 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35575-0x10015be3d850000, quorum=127.0.0.1:50343, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-11T16:23:44,172 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@47440849{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T16:23:44,173 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@41a23710{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-11T16:23:44,173 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-11T16:23:44,173 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@25ae6aaf{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-11T16:23:44,173 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5d562032{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9849ee79-554d-04b7-b255-e4857b767920/hadoop.log.dir/,STOPPED}
2024-12-11T16:23:44,180 WARN  [BP-1424680021-172.17.0.2-1733934216809 heartbeating to localhost/127.0.0.1:34957 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-11T16:23:44,180 WARN  [BP-1424680021-172.17.0.2-1733934216809 heartbeating to localhost/127.0.0.1:34957 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1424680021-172.17.0.2-1733934216809 (Datanode Uuid 0ac6d820-d9ff-4f11-892b-d93f697edbd9) service to localhost/127.0.0.1:34957
2024-12-11T16:23:44,181 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9849ee79-554d-04b7-b255-e4857b767920/cluster_b1b3091a-5088-7047-daf8-a83f640e6f09/data/data3/current/BP-1424680021-172.17.0.2-1733934216809 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T16:23:44,181 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-11T16:23:44,183 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-11T16:23:44,183 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-11T16:23:44,183 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9849ee79-554d-04b7-b255-e4857b767920/cluster_b1b3091a-5088-7047-daf8-a83f640e6f09/data/data4/current/BP-1424680021-172.17.0.2-1733934216809 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T16:23:44,216 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@21d2e33a{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T16:23:44,216 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@43aec6f9{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-11T16:23:44,216 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-11T16:23:44,217 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5383b5e5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-11T16:23:44,217 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4e62fea{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9849ee79-554d-04b7-b255-e4857b767920/hadoop.log.dir/,STOPPED}
2024-12-11T16:23:44,231 WARN  [BP-1424680021-172.17.0.2-1733934216809 heartbeating to localhost/127.0.0.1:34957 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-11T16:23:44,231 WARN  [BP-1424680021-172.17.0.2-1733934216809 heartbeating to localhost/127.0.0.1:34957 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1424680021-172.17.0.2-1733934216809 (Datanode Uuid f150b863-5e60-4a1e-b798-b37dae44fa90) service to localhost/127.0.0.1:34957
2024-12-11T16:23:44,231 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-11T16:23:44,233 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-11T16:23:44,233 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-11T16:23:44,234 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9849ee79-554d-04b7-b255-e4857b767920/cluster_b1b3091a-5088-7047-daf8-a83f640e6f09/data/data1/current/BP-1424680021-172.17.0.2-1733934216809 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T16:23:44,237 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9849ee79-554d-04b7-b255-e4857b767920/cluster_b1b3091a-5088-7047-daf8-a83f640e6f09/data/data2/current/BP-1424680021-172.17.0.2-1733934216809 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T16:23:44,242 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5b8ba39e{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-11T16:23:44,243 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2d2a6b1f{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-11T16:23:44,243 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-11T16:23:44,244 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5ee8b880{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-11T16:23:44,247 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@51945247{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9849ee79-554d-04b7-b255-e4857b767920/hadoop.log.dir/,STOPPED}
2024-12-11T16:23:44,288 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-12-11T16:23:44,350 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-12-11T16:23:44,350 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-12-11T16:23:44,350 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9849ee79-554d-04b7-b255-e4857b767920/hadoop.log.dir so I do NOT create it in target/test-data/53036128-f216-6ce6-44c0-30b05c226f86
2024-12-11T16:23:44,350 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9849ee79-554d-04b7-b255-e4857b767920/hadoop.tmp.dir so I do NOT create it in target/test-data/53036128-f216-6ce6-44c0-30b05c226f86
2024-12-11T16:23:44,351 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/cluster_7052d23d-6944-2d6a-b15d-561f403fc1d1, deleteOnExit=true
2024-12-11T16:23:44,351 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-12-11T16:23:44,351 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/test.cache.data in system properties and HBase conf
2024-12-11T16:23:44,351 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/hadoop.tmp.dir in system properties and HBase conf
2024-12-11T16:23:44,351 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/hadoop.log.dir in system properties and HBase conf
2024-12-11T16:23:44,351 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/mapreduce.cluster.local.dir in system properties and HBase conf
2024-12-11T16:23:44,351 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-12-11T16:23:44,351 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-12-11T16:23:44,352 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-11T16:23:44,354 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-12-11T16:23:44,354 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-12-11T16:23:44,354 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-12-11T16:23:44,354 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-11T16:23:44,355 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-12-11T16:23:44,355 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-12-11T16:23:44,355 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-11T16:23:44,355 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-11T16:23:44,356 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-12-11T16:23:44,356 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/nfs.dump.dir in system properties and HBase conf
2024-12-11T16:23:44,356 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/java.io.tmpdir in system properties and HBase conf
2024-12-11T16:23:44,356 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-11T16:23:44,357 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-12-11T16:23:44,357 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-12-11T16:23:44,390 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-11T16:23:44,717 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-11T16:23:44,730 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-11T16:23:44,776 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-11T16:23:44,776 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-11T16:23:44,776 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-11T16:23:44,780 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-11T16:23:44,781 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@f31077{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/hadoop.log.dir/,AVAILABLE}
2024-12-11T16:23:44,783 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@c0b5dad{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-11T16:23:44,921 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6668e7cf{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/java.io.tmpdir/jetty-localhost-39171-hadoop-hdfs-3_4_1-tests_jar-_-any-14833810946059043856/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-11T16:23:44,922 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@a936cd8{HTTP/1.1, (http/1.1)}{localhost:39171}
2024-12-11T16:23:44,922 INFO  [Time-limited test {}] server.Server(415): Started @115592ms
2024-12-11T16:23:44,940 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-11T16:23:45,192 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-11T16:23:45,196 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-11T16:23:45,197 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-11T16:23:45,197 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-11T16:23:45,197 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-11T16:23:45,198 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5c11a3d2{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/hadoop.log.dir/,AVAILABLE}
2024-12-11T16:23:45,199 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2c5fd7de{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-11T16:23:45,333 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@390e4b46{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/java.io.tmpdir/jetty-localhost-45767-hadoop-hdfs-3_4_1-tests_jar-_-any-3692189474600130775/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T16:23:45,334 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@78ec1620{HTTP/1.1, (http/1.1)}{localhost:45767}
2024-12-11T16:23:45,334 INFO  [Time-limited test {}] server.Server(415): Started @116004ms
2024-12-11T16:23:45,336 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-11T16:23:45,412 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-11T16:23:45,416 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-11T16:23:45,419 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-11T16:23:45,420 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-11T16:23:45,420 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-11T16:23:45,420 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@74afbef6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/hadoop.log.dir/,AVAILABLE}
2024-12-11T16:23:45,421 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@78ac40e9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-11T16:23:45,534 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5def8371{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/java.io.tmpdir/jetty-localhost-40993-hadoop-hdfs-3_4_1-tests_jar-_-any-5213105112205470731/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T16:23:45,535 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@77f8647e{HTTP/1.1, (http/1.1)}{localhost:40993}
2024-12-11T16:23:45,535 INFO  [Time-limited test {}] server.Server(415): Started @116205ms
2024-12-11T16:23:45,536 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-11T16:23:46,147 WARN  [Thread-671 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/cluster_7052d23d-6944-2d6a-b15d-561f403fc1d1/data/data1/current/BP-1552246360-172.17.0.2-1733934224410/current, will proceed with Du for space computation calculation, 
2024-12-11T16:23:46,155 WARN  [Thread-672 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/cluster_7052d23d-6944-2d6a-b15d-561f403fc1d1/data/data2/current/BP-1552246360-172.17.0.2-1733934224410/current, will proceed with Du for space computation calculation, 
2024-12-11T16:23:46,196 WARN  [Thread-636 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-11T16:23:46,201 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xad25a2b167716c1b with lease ID 0x625a5ee7777b3bed: Processing first storage report for DS-ea0e888f-b34d-4dd3-bb95-5725242e322a from datanode DatanodeRegistration(127.0.0.1:44567, datanodeUuid=9bef2747-4727-42cc-97c6-d783b2288746, infoPort=44269, infoSecurePort=0, ipcPort=38073, storageInfo=lv=-57;cid=testClusterID;nsid=1545217681;c=1733934224410)
2024-12-11T16:23:46,202 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xad25a2b167716c1b with lease ID 0x625a5ee7777b3bed: from storage DS-ea0e888f-b34d-4dd3-bb95-5725242e322a node DatanodeRegistration(127.0.0.1:44567, datanodeUuid=9bef2747-4727-42cc-97c6-d783b2288746, infoPort=44269, infoSecurePort=0, ipcPort=38073, storageInfo=lv=-57;cid=testClusterID;nsid=1545217681;c=1733934224410), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-11T16:23:46,202 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xad25a2b167716c1b with lease ID 0x625a5ee7777b3bed: Processing first storage report for DS-32a65b72-eee9-48c2-bd18-9086c45516fc from datanode DatanodeRegistration(127.0.0.1:44567, datanodeUuid=9bef2747-4727-42cc-97c6-d783b2288746, infoPort=44269, infoSecurePort=0, ipcPort=38073, storageInfo=lv=-57;cid=testClusterID;nsid=1545217681;c=1733934224410)
2024-12-11T16:23:46,202 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xad25a2b167716c1b with lease ID 0x625a5ee7777b3bed: from storage DS-32a65b72-eee9-48c2-bd18-9086c45516fc node DatanodeRegistration(127.0.0.1:44567, datanodeUuid=9bef2747-4727-42cc-97c6-d783b2288746, infoPort=44269, infoSecurePort=0, ipcPort=38073, storageInfo=lv=-57;cid=testClusterID;nsid=1545217681;c=1733934224410), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-11T16:23:46,415 WARN  [Thread-683 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/cluster_7052d23d-6944-2d6a-b15d-561f403fc1d1/data/data3/current/BP-1552246360-172.17.0.2-1733934224410/current, will proceed with Du for space computation calculation, 
2024-12-11T16:23:46,424 WARN  [Thread-684 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/cluster_7052d23d-6944-2d6a-b15d-561f403fc1d1/data/data4/current/BP-1552246360-172.17.0.2-1733934224410/current, will proceed with Du for space computation calculation, 
2024-12-11T16:23:46,465 WARN  [Thread-659 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-11T16:23:46,469 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x17e406b3fb490dff with lease ID 0x625a5ee7777b3bee: Processing first storage report for DS-226c2206-3817-4f6d-b23c-9dac8d5d6b69 from datanode DatanodeRegistration(127.0.0.1:38801, datanodeUuid=3250d228-4a4c-40f3-8d16-2b51b2a499ea, infoPort=44809, infoSecurePort=0, ipcPort=38099, storageInfo=lv=-57;cid=testClusterID;nsid=1545217681;c=1733934224410)
2024-12-11T16:23:46,469 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x17e406b3fb490dff with lease ID 0x625a5ee7777b3bee: from storage DS-226c2206-3817-4f6d-b23c-9dac8d5d6b69 node DatanodeRegistration(127.0.0.1:38801, datanodeUuid=3250d228-4a4c-40f3-8d16-2b51b2a499ea, infoPort=44809, infoSecurePort=0, ipcPort=38099, storageInfo=lv=-57;cid=testClusterID;nsid=1545217681;c=1733934224410), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-12-11T16:23:46,469 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x17e406b3fb490dff with lease ID 0x625a5ee7777b3bee: Processing first storage report for DS-0fedcaca-d67f-4052-96cd-792e7942bfff from datanode DatanodeRegistration(127.0.0.1:38801, datanodeUuid=3250d228-4a4c-40f3-8d16-2b51b2a499ea, infoPort=44809, infoSecurePort=0, ipcPort=38099, storageInfo=lv=-57;cid=testClusterID;nsid=1545217681;c=1733934224410)
2024-12-11T16:23:46,469 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x17e406b3fb490dff with lease ID 0x625a5ee7777b3bee: from storage DS-0fedcaca-d67f-4052-96cd-792e7942bfff node DatanodeRegistration(127.0.0.1:38801, datanodeUuid=3250d228-4a4c-40f3-8d16-2b51b2a499ea, infoPort=44809, infoSecurePort=0, ipcPort=38099, storageInfo=lv=-57;cid=testClusterID;nsid=1545217681;c=1733934224410), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-11T16:23:46,555 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86
2024-12-11T16:23:46,558 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/cluster_7052d23d-6944-2d6a-b15d-561f403fc1d1/zookeeper_0, clientPort=53294, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/cluster_7052d23d-6944-2d6a-b15d-561f403fc1d1/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/cluster_7052d23d-6944-2d6a-b15d-561f403fc1d1/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-11T16:23:46,567 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=53294
2024-12-11T16:23:46,568 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T16:23:46,570 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T16:23:46,599 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38801 is added to blk_1073741825_1001 (size=7)
2024-12-11T16:23:46,599 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44567 is added to blk_1073741825_1001 (size=7)
2024-12-11T16:23:47,003 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee with version=8
2024-12-11T16:23:47,003 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/hbase-staging
2024-12-11T16:23:47,006 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/7446a65196a1:0 server-side Connection retries=45
2024-12-11T16:23:47,006 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-11T16:23:47,006 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-11T16:23:47,006 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-11T16:23:47,006 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-11T16:23:47,007 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-11T16:23:47,007 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-12-11T16:23:47,007 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-11T16:23:47,015 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36693
2024-12-11T16:23:47,017 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:36693 connecting to ZooKeeper ensemble=127.0.0.1:53294
2024-12-11T16:23:47,084 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-11T16:23:47,093 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:23:47,103 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:366930x0, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-11T16:23:47,106 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36693-0x10015be5b1a0000 connected
2024-12-11T16:23:47,120 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:23:47,135 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:23:47,136 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:23:47,210 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T16:23:47,213 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T16:23:47,216 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36693-0x10015be5b1a0000, quorum=127.0.0.1:53294, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-11T16:23:47,216 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee, hbase.cluster.distributed=false
2024-12-11T16:23:47,218 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36693-0x10015be5b1a0000, quorum=127.0.0.1:53294, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-11T16:23:47,227 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36693
2024-12-11T16:23:47,229 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36693
2024-12-11T16:23:47,229 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36693
2024-12-11T16:23:47,231 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36693
2024-12-11T16:23:47,231 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36693
2024-12-11T16:23:47,249 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/7446a65196a1:0 server-side Connection retries=45
2024-12-11T16:23:47,249 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-11T16:23:47,249 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-11T16:23:47,249 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-11T16:23:47,249 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-11T16:23:47,249 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-11T16:23:47,249 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-11T16:23:47,249 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-11T16:23:47,250 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45947
2024-12-11T16:23:47,251 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45947 connecting to ZooKeeper ensemble=127.0.0.1:53294
2024-12-11T16:23:47,252 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T16:23:47,254 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T16:23:47,277 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:459470x0, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-11T16:23:47,277 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:459470x0, quorum=127.0.0.1:53294, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-11T16:23:47,278 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45947-0x10015be5b1a0001 connected
2024-12-11T16:23:47,278 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-11T16:23:47,283 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-11T16:23:47,284 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45947-0x10015be5b1a0001, quorum=127.0.0.1:53294, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-11T16:23:47,285 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45947-0x10015be5b1a0001, quorum=127.0.0.1:53294, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-11T16:23:47,287 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45947
2024-12-11T16:23:47,287 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45947
2024-12-11T16:23:47,287 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45947
2024-12-11T16:23:47,288 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45947
2024-12-11T16:23:47,289 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45947
2024-12-11T16:23:47,300 DEBUG [M:0;7446a65196a1:36693 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;7446a65196a1:36693
2024-12-11T16:23:47,300 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/7446a65196a1,36693,1733934227006
2024-12-11T16:23:47,327 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45947-0x10015be5b1a0001, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-11T16:23:47,327 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36693-0x10015be5b1a0000, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-11T16:23:47,327 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36693-0x10015be5b1a0000, quorum=127.0.0.1:53294, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/7446a65196a1,36693,1733934227006
2024-12-11T16:23:47,341 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45947-0x10015be5b1a0001, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-11T16:23:47,341 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36693-0x10015be5b1a0000, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:23:47,341 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45947-0x10015be5b1a0001, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:23:47,341 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36693-0x10015be5b1a0000, quorum=127.0.0.1:53294, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-11T16:23:47,342 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/7446a65196a1,36693,1733934227006 from backup master directory
2024-12-11T16:23:47,351 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45947-0x10015be5b1a0001, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-11T16:23:47,352 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36693-0x10015be5b1a0000, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/7446a65196a1,36693,1733934227006
2024-12-11T16:23:47,352 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36693-0x10015be5b1a0000, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-11T16:23:47,352 WARN  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-11T16:23:47,352 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=7446a65196a1,36693,1733934227006
2024-12-11T16:23:47,357 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/hbase.id] with ID: 6250230b-b09d-458a-a8c4-eb84fb89c3b8
2024-12-11T16:23:47,357 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/.tmp/hbase.id
2024-12-11T16:23:47,364 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38801 is added to blk_1073741826_1002 (size=42)
2024-12-11T16:23:47,365 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44567 is added to blk_1073741826_1002 (size=42)
2024-12-11T16:23:47,365 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/.tmp/hbase.id]:[hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/hbase.id]
2024-12-11T16:23:47,378 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T16:23:47,378 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-12-11T16:23:47,379 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms.
2024-12-11T16:23:47,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45947-0x10015be5b1a0001, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:23:47,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36693-0x10015be5b1a0000, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:23:47,419 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44567 is added to blk_1073741827_1003 (size=196)
2024-12-11T16:23:47,420 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38801 is added to blk_1073741827_1003 (size=196)
2024-12-11T16:23:47,420 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-11T16:23:47,421 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-11T16:23:47,421 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-11T16:23:47,428 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44567 is added to blk_1073741828_1004 (size=1189)
2024-12-11T16:23:47,429 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38801 is added to blk_1073741828_1004 (size=1189)
2024-12-11T16:23:47,429 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/MasterData/data/master/store
2024-12-11T16:23:47,436 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44567 is added to blk_1073741829_1005 (size=34)
2024-12-11T16:23:47,437 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38801 is added to blk_1073741829_1005 (size=34)
2024-12-11T16:23:47,437 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T16:23:47,437 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-11T16:23:47,437 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T16:23:47,438 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T16:23:47,438 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-11T16:23:47,438 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T16:23:47,438 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T16:23:47,438 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733934227437Disabling compacts and flushes for region at 1733934227437Disabling writes for close at 1733934227438 (+1 ms)Writing region close event to WAL at 1733934227438Closed at 1733934227438
2024-12-11T16:23:47,439 WARN  [master/7446a65196a1:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/MasterData/data/master/store/.initializing
2024-12-11T16:23:47,439 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/MasterData/WALs/7446a65196a1,36693,1733934227006
2024-12-11T16:23:47,442 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7446a65196a1%2C36693%2C1733934227006, suffix=, logDir=hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/MasterData/WALs/7446a65196a1,36693,1733934227006, archiveDir=hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/MasterData/oldWALs, maxLogs=10
2024-12-11T16:23:47,442 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7446a65196a1%2C36693%2C1733934227006.1733934227442
2024-12-11T16:23:47,447 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/MasterData/WALs/7446a65196a1,36693,1733934227006/7446a65196a1%2C36693%2C1733934227006.1733934227442
2024-12-11T16:23:47,453 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44269:44269),(127.0.0.1/127.0.0.1:44809:44809)]
2024-12-11T16:23:47,455 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-11T16:23:47,455 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T16:23:47,455 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T16:23:47,455 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T16:23:47,458 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-11T16:23:47,459 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-11T16:23:47,459 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:23:47,460 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T16:23:47,460 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-11T16:23:47,461 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-11T16:23:47,462 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:23:47,462 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-11T16:23:47,462 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-11T16:23:47,464 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-11T16:23:47,464 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:23:47,464 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-11T16:23:47,464 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-11T16:23:47,466 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-11T16:23:47,466 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:23:47,467 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-11T16:23:47,467 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T16:23:47,468 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-11T16:23:47,468 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-11T16:23:47,470 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T16:23:47,470 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T16:23:47,470 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-11T16:23:47,471 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T16:23:47,473 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-11T16:23:47,474 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=852757, jitterRate=0.08433675765991211}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-11T16:23:47,475 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1733934227455Initializing all the Stores at 1733934227456 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733934227456Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733934227458 (+2 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733934227458Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733934227458Cleaning up temporary data from old regions at 1733934227470 (+12 ms)Region opened successfully at 1733934227475 (+5 ms)
2024-12-11T16:23:47,478 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-11T16:23:47,482 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37f9d317, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7446a65196a1/172.17.0.2:0
2024-12-11T16:23:47,483 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-12-11T16:23:47,483 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-11T16:23:47,484 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-11T16:23:47,484 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-11T16:23:47,484 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-12-11T16:23:47,485 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-12-11T16:23:47,485 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-11T16:23:47,488 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-11T16:23:47,490 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36693-0x10015be5b1a0000, quorum=127.0.0.1:53294, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-11T16:23:47,510 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-12-11T16:23:47,510 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-11T16:23:47,511 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36693-0x10015be5b1a0000, quorum=127.0.0.1:53294, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-11T16:23:47,518 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-12-11T16:23:47,518 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-11T16:23:47,519 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36693-0x10015be5b1a0000, quorum=127.0.0.1:53294, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-11T16:23:47,526 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-12-11T16:23:47,528 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36693-0x10015be5b1a0000, quorum=127.0.0.1:53294, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-11T16:23:47,535 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-12-11T16:23:47,537 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36693-0x10015be5b1a0000, quorum=127.0.0.1:53294, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-11T16:23:47,543 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-11T16:23:47,551 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45947-0x10015be5b1a0001, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-11T16:23:47,551 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36693-0x10015be5b1a0000, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-11T16:23:47,551 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45947-0x10015be5b1a0001, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:23:47,551 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36693-0x10015be5b1a0000, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:23:47,553 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=7446a65196a1,36693,1733934227006, sessionid=0x10015be5b1a0000, setting cluster-up flag (Was=false)
2024-12-11T16:23:47,568 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36693-0x10015be5b1a0000, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:23:47,568 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45947-0x10015be5b1a0001, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:23:47,593 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-11T16:23:47,594 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=7446a65196a1,36693,1733934227006
2024-12-11T16:23:47,610 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36693-0x10015be5b1a0000, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:23:47,610 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45947-0x10015be5b1a0001, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:23:47,635 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-11T16:23:47,637 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=7446a65196a1,36693,1733934227006
2024-12-11T16:23:47,639 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-12-11T16:23:47,651 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-12-11T16:23:47,651 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-12-11T16:23:47,652 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-11T16:23:47,652 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 7446a65196a1,36693,1733934227006
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-11T16:23:47,654 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/7446a65196a1:0, corePoolSize=5, maxPoolSize=5
2024-12-11T16:23:47,654 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/7446a65196a1:0, corePoolSize=5, maxPoolSize=5
2024-12-11T16:23:47,654 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/7446a65196a1:0, corePoolSize=5, maxPoolSize=5
2024-12-11T16:23:47,654 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/7446a65196a1:0, corePoolSize=5, maxPoolSize=5
2024-12-11T16:23:47,654 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/7446a65196a1:0, corePoolSize=10, maxPoolSize=10
2024-12-11T16:23:47,654 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:23:47,655 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/7446a65196a1:0, corePoolSize=2, maxPoolSize=2
2024-12-11T16:23:47,655 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:23:47,656 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733934257656
2024-12-11T16:23:47,656 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-11T16:23:47,656 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-11T16:23:47,656 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-11T16:23:47,657 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-11T16:23:47,657 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-11T16:23:47,657 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-11T16:23:47,657 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-11T16:23:47,658 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-11T16:23:47,658 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-11T16:23:47,658 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-11T16:23:47,658 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-12-11T16:23:47,658 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-11T16:23:47,659 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-11T16:23:47,659 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-11T16:23:47,659 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/7446a65196a1:0:becomeActiveMaster-HFileCleaner.large.0-1733934227659,5,FailOnTimeoutGroup]
2024-12-11T16:23:47,659 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:23:47,660 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-11T16:23:47,670 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/7446a65196a1:0:becomeActiveMaster-HFileCleaner.small.0-1733934227659,5,FailOnTimeoutGroup]
2024-12-11T16:23:47,670 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-11T16:23:47,670 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-11T16:23:47,670 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-11T16:23:47,670 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-11T16:23:47,698 INFO  [RS:0;7446a65196a1:45947 {}] regionserver.HRegionServer(746): ClusterId : 6250230b-b09d-458a-a8c4-eb84fb89c3b8
2024-12-11T16:23:47,698 DEBUG [RS:0;7446a65196a1:45947 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-11T16:23:47,700 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38801 is added to blk_1073741831_1007 (size=1321)
2024-12-11T16:23:47,709 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44567 is added to blk_1073741831_1007 (size=1321)
2024-12-11T16:23:47,710 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-12-11T16:23:47,710 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee
2024-12-11T16:23:47,718 DEBUG [RS:0;7446a65196a1:45947 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-11T16:23:47,718 DEBUG [RS:0;7446a65196a1:45947 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-11T16:23:47,728 DEBUG [RS:0;7446a65196a1:45947 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-11T16:23:47,729 DEBUG [RS:0;7446a65196a1:45947 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18ef1e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7446a65196a1/172.17.0.2:0
2024-12-11T16:23:47,740 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44567 is added to blk_1073741832_1008 (size=32)
2024-12-11T16:23:47,746 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38801 is added to blk_1073741832_1008 (size=32)
2024-12-11T16:23:47,746 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T16:23:47,751 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-11T16:23:47,753 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-11T16:23:47,753 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:23:47,755 DEBUG [RS:0;7446a65196a1:45947 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;7446a65196a1:45947
2024-12-11T16:23:47,755 INFO  [RS:0;7446a65196a1:45947 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-12-11T16:23:47,755 INFO  [RS:0;7446a65196a1:45947 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-12-11T16:23:47,755 DEBUG [RS:0;7446a65196a1:45947 {}] regionserver.HRegionServer(832): About to register with Master.
2024-12-11T16:23:47,757 INFO  [RS:0;7446a65196a1:45947 {}] regionserver.HRegionServer(2659): reportForDuty to master=7446a65196a1,36693,1733934227006 with port=45947, startcode=1733934227248
2024-12-11T16:23:47,757 DEBUG [RS:0;7446a65196a1:45947 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-11T16:23:47,760 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T16:23:47,762 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-11T16:23:47,766 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-11T16:23:47,766 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:23:47,767 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T16:23:47,768 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-11T16:23:47,781 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-11T16:23:47,781 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:23:47,782 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T16:23:47,783 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-11T16:23:47,784 INFO  [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56179, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService
2024-12-11T16:23:47,785 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36693 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 7446a65196a1,45947,1733934227248
2024-12-11T16:23:47,785 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36693 {}] master.ServerManager(517): Registering regionserver=7446a65196a1,45947,1733934227248
2024-12-11T16:23:47,785 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-11T16:23:47,785 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:23:47,786 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T16:23:47,787 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-11T16:23:47,788 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/hbase/meta/1588230740
2024-12-11T16:23:47,789 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/hbase/meta/1588230740
2024-12-11T16:23:47,791 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-11T16:23:47,791 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-11T16:23:47,791 DEBUG [RS:0;7446a65196a1:45947 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee
2024-12-11T16:23:47,791 DEBUG [RS:0;7446a65196a1:45947 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:34525
2024-12-11T16:23:47,791 DEBUG [RS:0;7446a65196a1:45947 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-12-11T16:23:47,792 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-11T16:23:47,794 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-11T16:23:47,797 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-11T16:23:47,798 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=735533, jitterRate=-0.06472182273864746}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-11T16:23:47,799 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1733934227746Initializing all the Stores at 1733934227748 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733934227748Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733934227751 (+3 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733934227751Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733934227751Cleaning up temporary data from old regions at 1733934227791 (+40 ms)Region opened successfully at 1733934227799 (+8 ms)
2024-12-11T16:23:47,799 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-11T16:23:47,800 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-11T16:23:47,800 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-11T16:23:47,800 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-11T16:23:47,800 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-11T16:23:47,801 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-11T16:23:47,801 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733934227799Disabling compacts and flushes for region at 1733934227799Disabling writes for close at 1733934227800 (+1 ms)Writing region close event to WAL at 1733934227801 (+1 ms)Closed at 1733934227801
2024-12-11T16:23:47,803 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36693-0x10015be5b1a0000, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-11T16:23:47,803 DEBUG [RS:0;7446a65196a1:45947 {}] zookeeper.ZKUtil(111): regionserver:45947-0x10015be5b1a0001, quorum=127.0.0.1:53294, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/7446a65196a1,45947,1733934227248
2024-12-11T16:23:47,803 WARN  [RS:0;7446a65196a1:45947 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-11T16:23:47,804 INFO  [RS:0;7446a65196a1:45947 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-11T16:23:47,804 DEBUG [RS:0;7446a65196a1:45947 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248
2024-12-11T16:23:47,805 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-11T16:23:47,805 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-12-11T16:23:47,805 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-11T16:23:47,808 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-11T16:23:47,811 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-11T16:23:47,815 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [7446a65196a1,45947,1733934227248]
2024-12-11T16:23:47,821 INFO  [RS:0;7446a65196a1:45947 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-11T16:23:47,827 INFO  [RS:0;7446a65196a1:45947 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-11T16:23:47,827 INFO  [RS:0;7446a65196a1:45947 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-11T16:23:47,827 INFO  [RS:0;7446a65196a1:45947 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-11T16:23:47,828 INFO  [RS:0;7446a65196a1:45947 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-12-11T16:23:47,829 INFO  [RS:0;7446a65196a1:45947 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-12-11T16:23:47,829 INFO  [RS:0;7446a65196a1:45947 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-11T16:23:47,829 DEBUG [RS:0;7446a65196a1:45947 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:23:47,829 DEBUG [RS:0;7446a65196a1:45947 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:23:47,829 DEBUG [RS:0;7446a65196a1:45947 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:23:47,829 DEBUG [RS:0;7446a65196a1:45947 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:23:47,829 DEBUG [RS:0;7446a65196a1:45947 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:23:47,829 DEBUG [RS:0;7446a65196a1:45947 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/7446a65196a1:0, corePoolSize=2, maxPoolSize=2
2024-12-11T16:23:47,829 DEBUG [RS:0;7446a65196a1:45947 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:23:47,829 DEBUG [RS:0;7446a65196a1:45947 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:23:47,830 DEBUG [RS:0;7446a65196a1:45947 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:23:47,830 DEBUG [RS:0;7446a65196a1:45947 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:23:47,830 DEBUG [RS:0;7446a65196a1:45947 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:23:47,830 DEBUG [RS:0;7446a65196a1:45947 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:23:47,830 DEBUG [RS:0;7446a65196a1:45947 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/7446a65196a1:0, corePoolSize=3, maxPoolSize=3
2024-12-11T16:23:47,830 DEBUG [RS:0;7446a65196a1:45947 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/7446a65196a1:0, corePoolSize=3, maxPoolSize=3
2024-12-11T16:23:47,831 INFO  [RS:0;7446a65196a1:45947 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-11T16:23:47,831 INFO  [RS:0;7446a65196a1:45947 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-11T16:23:47,831 INFO  [RS:0;7446a65196a1:45947 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-11T16:23:47,831 INFO  [RS:0;7446a65196a1:45947 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-11T16:23:47,831 INFO  [RS:0;7446a65196a1:45947 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-11T16:23:47,831 INFO  [RS:0;7446a65196a1:45947 {}] hbase.ChoreService(168): Chore ScheduledChore name=7446a65196a1,45947,1733934227248-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-11T16:23:47,847 INFO  [RS:0;7446a65196a1:45947 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-11T16:23:47,847 INFO  [RS:0;7446a65196a1:45947 {}] hbase.ChoreService(168): Chore ScheduledChore name=7446a65196a1,45947,1733934227248-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-11T16:23:47,847 INFO  [RS:0;7446a65196a1:45947 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T16:23:47,847 INFO  [RS:0;7446a65196a1:45947 {}] regionserver.Replication(171): 7446a65196a1,45947,1733934227248 started
2024-12-11T16:23:47,868 INFO  [RS:0;7446a65196a1:45947 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T16:23:47,868 INFO  [RS:0;7446a65196a1:45947 {}] regionserver.HRegionServer(1482): Serving as 7446a65196a1,45947,1733934227248, RpcServer on 7446a65196a1/172.17.0.2:45947, sessionid=0x10015be5b1a0001
2024-12-11T16:23:47,868 DEBUG [RS:0;7446a65196a1:45947 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-11T16:23:47,868 DEBUG [RS:0;7446a65196a1:45947 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 7446a65196a1,45947,1733934227248
2024-12-11T16:23:47,868 DEBUG [RS:0;7446a65196a1:45947 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7446a65196a1,45947,1733934227248'
2024-12-11T16:23:47,868 DEBUG [RS:0;7446a65196a1:45947 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-11T16:23:47,869 DEBUG [RS:0;7446a65196a1:45947 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-11T16:23:47,870 DEBUG [RS:0;7446a65196a1:45947 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-11T16:23:47,870 DEBUG [RS:0;7446a65196a1:45947 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-11T16:23:47,870 DEBUG [RS:0;7446a65196a1:45947 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 7446a65196a1,45947,1733934227248
2024-12-11T16:23:47,870 DEBUG [RS:0;7446a65196a1:45947 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7446a65196a1,45947,1733934227248'
2024-12-11T16:23:47,870 DEBUG [RS:0;7446a65196a1:45947 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-11T16:23:47,871 DEBUG [RS:0;7446a65196a1:45947 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-11T16:23:47,871 DEBUG [RS:0;7446a65196a1:45947 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-11T16:23:47,871 INFO  [RS:0;7446a65196a1:45947 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-11T16:23:47,871 INFO  [RS:0;7446a65196a1:45947 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-11T16:23:47,961 WARN  [7446a65196a1:36693 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-12-11T16:23:47,979 INFO  [RS:0;7446a65196a1:45947 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7446a65196a1%2C45947%2C1733934227248, suffix=, logDir=hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248, archiveDir=hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/oldWALs, maxLogs=32
2024-12-11T16:23:47,981 INFO  [RS:0;7446a65196a1:45947 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7446a65196a1%2C45947%2C1733934227248.1733934227980
2024-12-11T16:23:47,990 INFO  [RS:0;7446a65196a1:45947 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.1733934227980
2024-12-11T16:23:47,995 DEBUG [RS:0;7446a65196a1:45947 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44269:44269),(127.0.0.1/127.0.0.1:44809:44809)]
2024-12-11T16:23:48,212 DEBUG [7446a65196a1:36693 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-12-11T16:23:48,212 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=7446a65196a1,45947,1733934227248
2024-12-11T16:23:48,214 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7446a65196a1,45947,1733934227248, state=OPENING
2024-12-11T16:23:48,224 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-11T16:23:48,235 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45947-0x10015be5b1a0001, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:23:48,235 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36693-0x10015be5b1a0000, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:23:48,235 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-11T16:23:48,235 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-11T16:23:48,235 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-11T16:23:48,235 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=7446a65196a1,45947,1733934227248}]
2024-12-11T16:23:48,389 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-11T16:23:48,391 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43383, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-11T16:23:48,396 INFO  [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-12-11T16:23:48,396 INFO  [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-11T16:23:48,398 INFO  [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7446a65196a1%2C45947%2C1733934227248.meta, suffix=.meta, logDir=hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248, archiveDir=hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/oldWALs, maxLogs=32
2024-12-11T16:23:48,399 INFO  [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
2024-12-11T16:23:48,406 INFO  [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
2024-12-11T16:23:48,410 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44809:44809),(127.0.0.1/127.0.0.1:44269:44269)]
2024-12-11T16:23:48,411 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-11T16:23:48,412 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-11T16:23:48,412 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-11T16:23:48,412 INFO  [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-11T16:23:48,412 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-11T16:23:48,412 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T16:23:48,412 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-12-11T16:23:48,412 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-12-11T16:23:48,415 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-11T16:23:48,416 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-11T16:23:48,416 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:23:48,417 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T16:23:48,417 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-11T16:23:48,418 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-11T16:23:48,418 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:23:48,419 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T16:23:48,419 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-11T16:23:48,420 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-11T16:23:48,420 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:23:48,421 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T16:23:48,421 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-11T16:23:48,422 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-11T16:23:48,422 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:23:48,422 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T16:23:48,422 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-11T16:23:48,423 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/hbase/meta/1588230740
2024-12-11T16:23:48,425 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/hbase/meta/1588230740
2024-12-11T16:23:48,426 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-11T16:23:48,426 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-11T16:23:48,427 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-11T16:23:48,429 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-11T16:23:48,430 INFO  [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=807842, jitterRate=0.02722473442554474}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-11T16:23:48,430 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-12-11T16:23:48,431 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1733934228412Writing region info on filesystem at 1733934228412Initializing all the Stores at 1733934228414 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733934228414Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733934228415 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733934228415Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733934228415Cleaning up temporary data from old regions at 1733934228426 (+11 ms)Running coprocessor post-open hooks at 1733934228430 (+4 ms)Region opened successfully at 1733934228431 (+1 ms)
2024-12-11T16:23:48,432 INFO  [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733934228389
2024-12-11T16:23:48,435 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-11T16:23:48,435 INFO  [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-12-11T16:23:48,436 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=7446a65196a1,45947,1733934227248
2024-12-11T16:23:48,437 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7446a65196a1,45947,1733934227248, state=OPEN
2024-12-11T16:23:48,494 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36693-0x10015be5b1a0000, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-11T16:23:48,494 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45947-0x10015be5b1a0001, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-11T16:23:48,494 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=7446a65196a1,45947,1733934227248
2024-12-11T16:23:48,494 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-11T16:23:48,494 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-11T16:23:48,499 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-12-11T16:23:48,499 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7446a65196a1,45947,1733934227248 in 259 msec
2024-12-11T16:23:48,503 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-12-11T16:23:48,503 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 694 msec
2024-12-11T16:23:48,504 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-11T16:23:48,504 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-12-11T16:23:48,506 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-11T16:23:48,506 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7446a65196a1,45947,1733934227248, seqNum=-1]
2024-12-11T16:23:48,506 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-11T16:23:48,508 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42497, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-11T16:23:48,516 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 873 msec
2024-12-11T16:23:48,516 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733934228516, completionTime=-1
2024-12-11T16:23:48,516 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-12-11T16:23:48,516 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-12-11T16:23:48,518 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-12-11T16:23:48,518 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733934288518
2024-12-11T16:23:48,519 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733934348518
2024-12-11T16:23:48,519 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec
2024-12-11T16:23:48,519 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7446a65196a1,36693,1733934227006-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-11T16:23:48,519 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7446a65196a1,36693,1733934227006-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T16:23:48,519 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7446a65196a1,36693,1733934227006-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T16:23:48,519 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-7446a65196a1:36693, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T16:23:48,519 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-11T16:23:48,519 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-12-11T16:23:48,522 DEBUG [master/7446a65196a1:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-12-11T16:23:48,524 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.172sec
2024-12-11T16:23:48,524 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-11T16:23:48,524 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-11T16:23:48,525 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-11T16:23:48,525 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-11T16:23:48,525 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-11T16:23:48,525 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7446a65196a1,36693,1733934227006-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-11T16:23:48,525 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7446a65196a1,36693,1733934227006-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-11T16:23:48,527 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-12-11T16:23:48,528 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-11T16:23:48,528 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7446a65196a1,36693,1733934227006-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T16:23:48,600 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74fac12f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-11T16:23:48,601 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7446a65196a1,36693,-1 for getting cluster id
2024-12-11T16:23:48,601 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-12-11T16:23:48,603 DEBUG [HMaster-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6250230b-b09d-458a-a8c4-eb84fb89c3b8'
2024-12-11T16:23:48,603 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-12-11T16:23:48,604 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6250230b-b09d-458a-a8c4-eb84fb89c3b8"

2024-12-11T16:23:48,604 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@92baeef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-11T16:23:48,604 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7446a65196a1,36693,-1]
2024-12-11T16:23:48,604 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-12-11T16:23:48,605 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T16:23:48,607 INFO  [HMaster-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38756, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-12-11T16:23:48,608 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8bac5bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-11T16:23:48,608 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-11T16:23:48,610 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7446a65196a1,45947,1733934227248, seqNum=-1]
2024-12-11T16:23:48,611 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-11T16:23:48,646 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50016, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-11T16:23:48,648 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=7446a65196a1,36693,1733934227006
2024-12-11T16:23:48,648 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T16:23:48,652 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-12-11T16:23:48,675 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/7446a65196a1:0 server-side Connection retries=45
2024-12-11T16:23:48,675 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-11T16:23:48,675 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-11T16:23:48,675 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-11T16:23:48,675 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-11T16:23:48,675 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-11T16:23:48,675 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-11T16:23:48,676 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-11T16:23:48,676 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42595
2024-12-11T16:23:48,678 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42595 connecting to ZooKeeper ensemble=127.0.0.1:53294
2024-12-11T16:23:48,679 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T16:23:48,681 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T16:23:48,726 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:425950x0, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-11T16:23:48,727 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:425950x0, quorum=127.0.0.1:53294, baseZNode=/hbase Set watcher on existing znode=/hbase/running
2024-12-11T16:23:48,727 DEBUG [pool-381-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT
2024-12-11T16:23:48,729 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-11T16:23:48,734 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42595-0x10015be5b1a0002 connected
2024-12-11T16:23:48,736 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-11T16:23:48,737 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:42595-0x10015be5b1a0002, quorum=127.0.0.1:53294, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-11T16:23:48,738 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42595-0x10015be5b1a0002, quorum=127.0.0.1:53294, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-11T16:23:48,743 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42595
2024-12-11T16:23:48,743 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42595
2024-12-11T16:23:48,744 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42595
2024-12-11T16:23:48,745 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42595
2024-12-11T16:23:48,745 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42595
2024-12-11T16:23:48,747 INFO  [RS:1;7446a65196a1:42595 {}] regionserver.HRegionServer(746): ClusterId : 6250230b-b09d-458a-a8c4-eb84fb89c3b8
2024-12-11T16:23:48,747 DEBUG [RS:1;7446a65196a1:42595 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-11T16:23:48,790 DEBUG [RS:1;7446a65196a1:42595 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-11T16:23:48,790 DEBUG [RS:1;7446a65196a1:42595 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-11T16:23:48,826 DEBUG [RS:1;7446a65196a1:42595 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-11T16:23:48,827 DEBUG [RS:1;7446a65196a1:42595 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d637cda, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7446a65196a1/172.17.0.2:0
2024-12-11T16:23:48,843 DEBUG [RS:1;7446a65196a1:42595 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;7446a65196a1:42595
2024-12-11T16:23:48,843 INFO  [RS:1;7446a65196a1:42595 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-12-11T16:23:48,843 INFO  [RS:1;7446a65196a1:42595 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-12-11T16:23:48,843 DEBUG [RS:1;7446a65196a1:42595 {}] regionserver.HRegionServer(832): About to register with Master.
2024-12-11T16:23:48,844 INFO  [RS:1;7446a65196a1:42595 {}] regionserver.HRegionServer(2659): reportForDuty to master=7446a65196a1,36693,1733934227006 with port=42595, startcode=1733934228675
2024-12-11T16:23:48,844 DEBUG [RS:1;7446a65196a1:42595 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-11T16:23:48,846 INFO  [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50097, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.3 (auth:SIMPLE), service=RegionServerStatusService
2024-12-11T16:23:48,847 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36693 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 7446a65196a1,42595,1733934228675
2024-12-11T16:23:48,847 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36693 {}] master.ServerManager(517): Registering regionserver=7446a65196a1,42595,1733934228675
2024-12-11T16:23:48,849 DEBUG [RS:1;7446a65196a1:42595 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee
2024-12-11T16:23:48,849 DEBUG [RS:1;7446a65196a1:42595 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:34525
2024-12-11T16:23:48,849 DEBUG [RS:1;7446a65196a1:42595 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-12-11T16:23:48,909 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36693-0x10015be5b1a0000, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-11T16:23:48,910 DEBUG [RS:1;7446a65196a1:42595 {}] zookeeper.ZKUtil(111): regionserver:42595-0x10015be5b1a0002, quorum=127.0.0.1:53294, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/7446a65196a1,42595,1733934228675
2024-12-11T16:23:48,910 WARN  [RS:1;7446a65196a1:42595 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-11T16:23:48,910 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [7446a65196a1,42595,1733934228675]
2024-12-11T16:23:48,910 INFO  [RS:1;7446a65196a1:42595 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-11T16:23:48,910 DEBUG [RS:1;7446a65196a1:42595 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675
2024-12-11T16:23:48,914 INFO  [RS:1;7446a65196a1:42595 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-11T16:23:48,916 INFO  [RS:1;7446a65196a1:42595 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-11T16:23:48,917 INFO  [RS:1;7446a65196a1:42595 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-11T16:23:48,917 INFO  [RS:1;7446a65196a1:42595 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-11T16:23:48,919 INFO  [RS:1;7446a65196a1:42595 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-12-11T16:23:48,920 INFO  [RS:1;7446a65196a1:42595 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-12-11T16:23:48,920 INFO  [RS:1;7446a65196a1:42595 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-11T16:23:48,920 DEBUG [RS:1;7446a65196a1:42595 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:23:48,920 DEBUG [RS:1;7446a65196a1:42595 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:23:48,920 DEBUG [RS:1;7446a65196a1:42595 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:23:48,920 DEBUG [RS:1;7446a65196a1:42595 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:23:48,920 DEBUG [RS:1;7446a65196a1:42595 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:23:48,920 DEBUG [RS:1;7446a65196a1:42595 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/7446a65196a1:0, corePoolSize=2, maxPoolSize=2
2024-12-11T16:23:48,920 DEBUG [RS:1;7446a65196a1:42595 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:23:48,920 DEBUG [RS:1;7446a65196a1:42595 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:23:48,920 DEBUG [RS:1;7446a65196a1:42595 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:23:48,920 DEBUG [RS:1;7446a65196a1:42595 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:23:48,920 DEBUG [RS:1;7446a65196a1:42595 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:23:48,920 DEBUG [RS:1;7446a65196a1:42595 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:23:48,921 DEBUG [RS:1;7446a65196a1:42595 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/7446a65196a1:0, corePoolSize=3, maxPoolSize=3
2024-12-11T16:23:48,921 DEBUG [RS:1;7446a65196a1:42595 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/7446a65196a1:0, corePoolSize=3, maxPoolSize=3
2024-12-11T16:23:48,924 INFO  [RS:1;7446a65196a1:42595 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-11T16:23:48,924 INFO  [RS:1;7446a65196a1:42595 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-11T16:23:48,924 INFO  [RS:1;7446a65196a1:42595 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-11T16:23:48,925 INFO  [RS:1;7446a65196a1:42595 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-11T16:23:48,925 INFO  [RS:1;7446a65196a1:42595 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-11T16:23:48,925 INFO  [RS:1;7446a65196a1:42595 {}] hbase.ChoreService(168): Chore ScheduledChore name=7446a65196a1,42595,1733934228675-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-11T16:23:48,941 INFO  [RS:1;7446a65196a1:42595 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-11T16:23:48,941 INFO  [RS:1;7446a65196a1:42595 {}] hbase.ChoreService(168): Chore ScheduledChore name=7446a65196a1,42595,1733934228675-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-11T16:23:48,942 INFO  [RS:1;7446a65196a1:42595 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T16:23:48,942 INFO  [RS:1;7446a65196a1:42595 {}] regionserver.Replication(171): 7446a65196a1,42595,1733934228675 started
2024-12-11T16:23:48,955 INFO  [RS:1;7446a65196a1:42595 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T16:23:48,955 INFO  [RS:1;7446a65196a1:42595 {}] regionserver.HRegionServer(1482): Serving as 7446a65196a1,42595,1733934228675, RpcServer on 7446a65196a1/172.17.0.2:42595, sessionid=0x10015be5b1a0002
2024-12-11T16:23:48,955 DEBUG [RS:1;7446a65196a1:42595 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-11T16:23:48,955 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(2882): Started new server=Thread[RS:1;7446a65196a1:42595,5,FailOnTimeoutGroup]
2024-12-11T16:23:48,955 DEBUG [RS:1;7446a65196a1:42595 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 7446a65196a1,42595,1733934228675
2024-12-11T16:23:48,955 DEBUG [RS:1;7446a65196a1:42595 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7446a65196a1,42595,1733934228675'
2024-12-11T16:23:48,955 DEBUG [RS:1;7446a65196a1:42595 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-11T16:23:48,956 INFO  [Time-limited test {}] wal.TestLogRolling(207): Replication=2
2024-12-11T16:23:48,956 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-12-11T16:23:48,956 DEBUG [RS:1;7446a65196a1:42595 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-11T16:23:48,957 DEBUG [RS:1;7446a65196a1:42595 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-11T16:23:48,957 DEBUG [RS:1;7446a65196a1:42595 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-11T16:23:48,957 DEBUG [RS:1;7446a65196a1:42595 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 7446a65196a1,42595,1733934228675
2024-12-11T16:23:48,957 DEBUG [RS:1;7446a65196a1:42595 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7446a65196a1,42595,1733934228675'
2024-12-11T16:23:48,957 DEBUG [RS:1;7446a65196a1:42595 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-11T16:23:48,957 DEBUG [RS:1;7446a65196a1:42595 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-11T16:23:48,957 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncConnectionImpl(321): The fetched master address is 7446a65196a1,36693,1733934227006
2024-12-11T16:23:48,958 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4159dbcd
2024-12-11T16:23:48,958 DEBUG [RS:1;7446a65196a1:42595 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-11T16:23:48,958 INFO  [RS:1;7446a65196a1:42595 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-11T16:23:48,958 INFO  [RS:1;7446a65196a1:42595 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-11T16:23:48,958 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-12-11T16:23:48,960 INFO  [HMaster-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38760, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-12-11T16:23:48,961 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-12-11T16:23:48,961 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-12-11T16:23:48,961 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRollOnDatanodeDeath', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-11T16:23:48,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath
2024-12-11T16:23:48,964 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_PRE_OPERATION
2024-12-11T16:23:48,964 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:23:48,964 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRollOnDatanodeDeath"
 procId is: 4
2024-12-11T16:23:48,966 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-12-11T16:23:48,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-11T16:23:48,974 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38801 is added to blk_1073741835_1011 (size=393)
2024-12-11T16:23:48,974 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44567 is added to blk_1073741835_1011 (size=393)
2024-12-11T16:23:48,976 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 96a752a5036dfbc63de411b3f9c85ba7, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1733934228960.96a752a5036dfbc63de411b3f9c85ba7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnDatanodeDeath', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee
2024-12-11T16:23:48,983 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44567 is added to blk_1073741836_1012 (size=76)
2024-12-11T16:23:48,984 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38801 is added to blk_1073741836_1012 (size=76)
2024-12-11T16:23:48,984 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1733934228960.96a752a5036dfbc63de411b3f9c85ba7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T16:23:48,984 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1722): Closing 96a752a5036dfbc63de411b3f9c85ba7, disabling compactions & flushes
2024-12-11T16:23:48,984 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1733934228960.96a752a5036dfbc63de411b3f9c85ba7.
2024-12-11T16:23:48,984 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1733934228960.96a752a5036dfbc63de411b3f9c85ba7.
2024-12-11T16:23:48,985 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1733934228960.96a752a5036dfbc63de411b3f9c85ba7. after waiting 0 ms
2024-12-11T16:23:48,985 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1733934228960.96a752a5036dfbc63de411b3f9c85ba7.
2024-12-11T16:23:48,985 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1733934228960.96a752a5036dfbc63de411b3f9c85ba7.
2024-12-11T16:23:48,985 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1676): Region close journal for 96a752a5036dfbc63de411b3f9c85ba7:
Waiting for close lock at 1733934228984Disabling compacts and flushes for region at 1733934228984Disabling writes for close at 1733934228985 (+1 ms)Writing region close event to WAL at 1733934228985Closed at 1733934228985
2024-12-11T16:23:48,986 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ADD_TO_META
2024-12-11T16:23:48,987 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnDatanodeDeath,,1733934228960.96a752a5036dfbc63de411b3f9c85ba7.","families":{"info":[{"qualifier":"regioninfo","vlen":75,"tag":[],"timestamp":"1733934228987"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733934228987"}]},"ts":"1733934228987"}
2024-12-11T16:23:48,990 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-12-11T16:23:48,991 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-12-11T16:23:48,992 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733934228991"}]},"ts":"1733934228991"}
2024-12-11T16:23:48,994 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLING in hbase:meta
2024-12-11T16:23:48,994 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=96a752a5036dfbc63de411b3f9c85ba7, ASSIGN}]
2024-12-11T16:23:48,995 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=96a752a5036dfbc63de411b3f9c85ba7, ASSIGN
2024-12-11T16:23:48,997 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=96a752a5036dfbc63de411b3f9c85ba7, ASSIGN; state=OFFLINE, location=7446a65196a1,45947,1733934227248; forceNewPlan=false, retain=false
2024-12-11T16:23:49,064 INFO  [RS:1;7446a65196a1:42595 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7446a65196a1%2C42595%2C1733934228675, suffix=, logDir=hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675, archiveDir=hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/oldWALs, maxLogs=32
2024-12-11T16:23:49,065 INFO  [RS:1;7446a65196a1:42595 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7446a65196a1%2C42595%2C1733934228675.1733934229064
2024-12-11T16:23:49,076 INFO  [RS:1;7446a65196a1:42595 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
2024-12-11T16:23:49,083 DEBUG [RS:1;7446a65196a1:42595 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44269:44269),(127.0.0.1/127.0.0.1:44809:44809)]
2024-12-11T16:23:49,148 INFO  [7446a65196a1:36693 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 
2024-12-11T16:23:49,148 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=96a752a5036dfbc63de411b3f9c85ba7, regionState=OPENING, regionLocation=7446a65196a1,45947,1733934227248
2024-12-11T16:23:49,154 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=96a752a5036dfbc63de411b3f9c85ba7, ASSIGN because future has completed
2024-12-11T16:23:49,155 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 96a752a5036dfbc63de411b3f9c85ba7, server=7446a65196a1,45947,1733934227248}]
2024-12-11T16:23:49,317 INFO  [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRollOnDatanodeDeath,,1733934228960.96a752a5036dfbc63de411b3f9c85ba7.
2024-12-11T16:23:49,318 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 96a752a5036dfbc63de411b3f9c85ba7, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1733934228960.96a752a5036dfbc63de411b3f9c85ba7.', STARTKEY => '', ENDKEY => ''}
2024-12-11T16:23:49,318 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnDatanodeDeath 96a752a5036dfbc63de411b3f9c85ba7
2024-12-11T16:23:49,318 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1733934228960.96a752a5036dfbc63de411b3f9c85ba7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T16:23:49,318 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 96a752a5036dfbc63de411b3f9c85ba7
2024-12-11T16:23:49,318 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 96a752a5036dfbc63de411b3f9c85ba7
2024-12-11T16:23:49,320 INFO  [StoreOpener-96a752a5036dfbc63de411b3f9c85ba7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 96a752a5036dfbc63de411b3f9c85ba7 
2024-12-11T16:23:49,322 INFO  [StoreOpener-96a752a5036dfbc63de411b3f9c85ba7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 96a752a5036dfbc63de411b3f9c85ba7 columnFamilyName info
2024-12-11T16:23:49,322 DEBUG [StoreOpener-96a752a5036dfbc63de411b3f9c85ba7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:23:49,322 INFO  [StoreOpener-96a752a5036dfbc63de411b3f9c85ba7-1 {}] regionserver.HStore(327): Store=96a752a5036dfbc63de411b3f9c85ba7/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-11T16:23:49,322 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 96a752a5036dfbc63de411b3f9c85ba7
2024-12-11T16:23:49,323 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7
2024-12-11T16:23:49,324 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7
2024-12-11T16:23:49,324 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 96a752a5036dfbc63de411b3f9c85ba7
2024-12-11T16:23:49,324 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 96a752a5036dfbc63de411b3f9c85ba7
2024-12-11T16:23:49,326 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 96a752a5036dfbc63de411b3f9c85ba7
2024-12-11T16:23:49,328 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-11T16:23:49,328 INFO  [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 96a752a5036dfbc63de411b3f9c85ba7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=768558, jitterRate=-0.02272886037826538}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-11T16:23:49,329 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 96a752a5036dfbc63de411b3f9c85ba7
2024-12-11T16:23:49,329 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 96a752a5036dfbc63de411b3f9c85ba7:
Running coprocessor pre-open hook at 1733934229318Writing region info on filesystem at 1733934229318Initializing all the Stores at 1733934229320 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733934229320Cleaning up temporary data from old regions at 1733934229324 (+4 ms)Running coprocessor post-open hooks at 1733934229329 (+5 ms)Region opened successfully at 1733934229329
2024-12-11T16:23:49,330 INFO  [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRollOnDatanodeDeath,,1733934228960.96a752a5036dfbc63de411b3f9c85ba7., pid=6, masterSystemTime=1733934229313
2024-12-11T16:23:49,333 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRollOnDatanodeDeath,,1733934228960.96a752a5036dfbc63de411b3f9c85ba7.
2024-12-11T16:23:49,333 INFO  [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRollOnDatanodeDeath,,1733934228960.96a752a5036dfbc63de411b3f9c85ba7.
2024-12-11T16:23:49,334 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=96a752a5036dfbc63de411b3f9c85ba7, regionState=OPEN, openSeqNum=2, regionLocation=7446a65196a1,45947,1733934227248
2024-12-11T16:23:49,336 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 96a752a5036dfbc63de411b3f9c85ba7, server=7446a65196a1,45947,1733934227248 because future has completed
2024-12-11T16:23:49,341 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-12-11T16:23:49,341 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 96a752a5036dfbc63de411b3f9c85ba7, server=7446a65196a1,45947,1733934227248 in 183 msec
2024-12-11T16:23:49,344 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-12-11T16:23:49,344 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=96a752a5036dfbc63de411b3f9c85ba7, ASSIGN in 347 msec
2024-12-11T16:23:49,345 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-12-11T16:23:49,345 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733934229345"}]},"ts":"1733934229345"}
2024-12-11T16:23:49,348 INFO  [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLED in hbase:meta
2024-12-11T16:23:49,349 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_POST_OPERATION
2024-12-11T16:23:49,351 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath in 388 msec
2024-12-11T16:23:53,915 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-11T16:23:53,916 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:23:53,928 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:23:53,932 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:23:53,933 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:23:53,943 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnDatanodeDeath'
2024-12-11T16:23:57,192 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-12-11T16:23:57,192 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-12-11T16:23:57,193 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath
2024-12-11T16:23:57,193 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath Metrics about Tables on a single HBase RegionServer
2024-12-11T16:23:57,193 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-11T16:23:57,194 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-12-11T16:23:57,194 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-12-11T16:23:57,194 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer
2024-12-11T16:23:59,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-11T16:23:59,031 INFO  [RPCClient-NioEventLoopGroup-4-11 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnDatanodeDeath completed
2024-12-11T16:23:59,031 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRollOnDatanodeDeath,, stopping at row=TestLogRolling-testLogRollOnDatanodeDeath ,, for max=2147483647 with caching=100
2024-12-11T16:23:59,034 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRollOnDatanodeDeath
2024-12-11T16:23:59,034 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRollOnDatanodeDeath,,1733934228960.96a752a5036dfbc63de411b3f9c85ba7.
2024-12-11T16:23:59,046 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-11T16:23:59,049 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-11T16:23:59,050 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-11T16:23:59,050 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-11T16:23:59,050 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-11T16:23:59,051 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1f5f345{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/hadoop.log.dir/,AVAILABLE}
2024-12-11T16:23:59,051 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@41b3c9b2{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-11T16:23:59,148 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5424ebd1{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/java.io.tmpdir/jetty-localhost-36885-hadoop-hdfs-3_4_1-tests_jar-_-any-10405911411408905905/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T16:23:59,148 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@70ae18b0{HTTP/1.1, (http/1.1)}{localhost:36885}
2024-12-11T16:23:59,148 INFO  [Time-limited test {}] server.Server(415): Started @129819ms
2024-12-11T16:23:59,149 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-11T16:23:59,202 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-11T16:23:59,205 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-11T16:23:59,206 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-11T16:23:59,206 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-11T16:23:59,207 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-11T16:23:59,209 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@20c0741c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/hadoop.log.dir/,AVAILABLE}
2024-12-11T16:23:59,209 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2d7d08f9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-11T16:23:59,326 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@768a1767{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/java.io.tmpdir/jetty-localhost-39839-hadoop-hdfs-3_4_1-tests_jar-_-any-9468644389450183772/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T16:23:59,326 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@44bcc293{HTTP/1.1, (http/1.1)}{localhost:39839}
2024-12-11T16:23:59,327 INFO  [Time-limited test {}] server.Server(415): Started @129997ms
2024-12-11T16:23:59,328 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-11T16:23:59,395 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-11T16:23:59,399 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-11T16:23:59,403 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-11T16:23:59,403 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-11T16:23:59,403 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-11T16:23:59,405 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4136d603{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/hadoop.log.dir/,AVAILABLE}
2024-12-11T16:23:59,405 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@75748ead{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-11T16:23:59,521 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7424ce2{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/java.io.tmpdir/jetty-localhost-40555-hadoop-hdfs-3_4_1-tests_jar-_-any-7985376717440963344/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T16:23:59,521 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7929c9ee{HTTP/1.1, (http/1.1)}{localhost:40555}
2024-12-11T16:23:59,521 INFO  [Time-limited test {}] server.Server(415): Started @130192ms
2024-12-11T16:23:59,523 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-11T16:24:00,280 WARN  [Thread-867 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/cluster_7052d23d-6944-2d6a-b15d-561f403fc1d1/data/data5/current/BP-1552246360-172.17.0.2-1733934224410/current, will proceed with Du for space computation calculation, 
2024-12-11T16:24:00,281 WARN  [Thread-868 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/cluster_7052d23d-6944-2d6a-b15d-561f403fc1d1/data/data6/current/BP-1552246360-172.17.0.2-1733934224410/current, will proceed with Du for space computation calculation, 
2024-12-11T16:24:00,299 WARN  [Thread-809 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-11T16:24:00,305 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa37a091511137fc3 with lease ID 0x625a5ee7777b3bef: Processing first storage report for DS-8e83c89e-91af-4855-9e85-7f0bb0de8fa8 from datanode DatanodeRegistration(127.0.0.1:39649, datanodeUuid=de133113-d741-4d77-878b-bfdb00c87c5e, infoPort=40753, infoSecurePort=0, ipcPort=41721, storageInfo=lv=-57;cid=testClusterID;nsid=1545217681;c=1733934224410)
2024-12-11T16:24:00,305 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa37a091511137fc3 with lease ID 0x625a5ee7777b3bef: from storage DS-8e83c89e-91af-4855-9e85-7f0bb0de8fa8 node DatanodeRegistration(127.0.0.1:39649, datanodeUuid=de133113-d741-4d77-878b-bfdb00c87c5e, infoPort=40753, infoSecurePort=0, ipcPort=41721, storageInfo=lv=-57;cid=testClusterID;nsid=1545217681;c=1733934224410), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-12-11T16:24:00,305 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa37a091511137fc3 with lease ID 0x625a5ee7777b3bef: Processing first storage report for DS-bd76ceb0-515c-4b53-b257-eece33d779aa from datanode DatanodeRegistration(127.0.0.1:39649, datanodeUuid=de133113-d741-4d77-878b-bfdb00c87c5e, infoPort=40753, infoSecurePort=0, ipcPort=41721, storageInfo=lv=-57;cid=testClusterID;nsid=1545217681;c=1733934224410)
2024-12-11T16:24:00,305 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa37a091511137fc3 with lease ID 0x625a5ee7777b3bef: from storage DS-bd76ceb0-515c-4b53-b257-eece33d779aa node DatanodeRegistration(127.0.0.1:39649, datanodeUuid=de133113-d741-4d77-878b-bfdb00c87c5e, infoPort=40753, infoSecurePort=0, ipcPort=41721, storageInfo=lv=-57;cid=testClusterID;nsid=1545217681;c=1733934224410), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-11T16:24:00,512 WARN  [Thread-880 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/cluster_7052d23d-6944-2d6a-b15d-561f403fc1d1/data/data8/current/BP-1552246360-172.17.0.2-1733934224410/current, will proceed with Du for space computation calculation, 
2024-12-11T16:24:00,512 WARN  [Thread-879 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/cluster_7052d23d-6944-2d6a-b15d-561f403fc1d1/data/data7/current/BP-1552246360-172.17.0.2-1733934224410/current, will proceed with Du for space computation calculation, 
2024-12-11T16:24:00,533 WARN  [Thread-831 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-11T16:24:00,539 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x52eef5844c7f2f8f with lease ID 0x625a5ee7777b3bf0: Processing first storage report for DS-d85b2b77-d19f-4302-be38-ce7ae4e72a21 from datanode DatanodeRegistration(127.0.0.1:35157, datanodeUuid=690ba6fc-5c14-4f6a-9c83-e257a13f29f3, infoPort=32897, infoSecurePort=0, ipcPort=42115, storageInfo=lv=-57;cid=testClusterID;nsid=1545217681;c=1733934224410)
2024-12-11T16:24:00,539 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x52eef5844c7f2f8f with lease ID 0x625a5ee7777b3bf0: from storage DS-d85b2b77-d19f-4302-be38-ce7ae4e72a21 node DatanodeRegistration(127.0.0.1:35157, datanodeUuid=690ba6fc-5c14-4f6a-9c83-e257a13f29f3, infoPort=32897, infoSecurePort=0, ipcPort=42115, storageInfo=lv=-57;cid=testClusterID;nsid=1545217681;c=1733934224410), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-11T16:24:00,539 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x52eef5844c7f2f8f with lease ID 0x625a5ee7777b3bf0: Processing first storage report for DS-1dba5253-7243-4307-818c-d29a89b05421 from datanode DatanodeRegistration(127.0.0.1:35157, datanodeUuid=690ba6fc-5c14-4f6a-9c83-e257a13f29f3, infoPort=32897, infoSecurePort=0, ipcPort=42115, storageInfo=lv=-57;cid=testClusterID;nsid=1545217681;c=1733934224410)
2024-12-11T16:24:00,539 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x52eef5844c7f2f8f with lease ID 0x625a5ee7777b3bf0: from storage DS-1dba5253-7243-4307-818c-d29a89b05421 node DatanodeRegistration(127.0.0.1:35157, datanodeUuid=690ba6fc-5c14-4f6a-9c83-e257a13f29f3, infoPort=32897, infoSecurePort=0, ipcPort=42115, storageInfo=lv=-57;cid=testClusterID;nsid=1545217681;c=1733934224410), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-11T16:24:00,648 WARN  [Thread-891 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/cluster_7052d23d-6944-2d6a-b15d-561f403fc1d1/data/data10/current/BP-1552246360-172.17.0.2-1733934224410/current, will proceed with Du for space computation calculation, 
2024-12-11T16:24:00,648 WARN  [Thread-890 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/cluster_7052d23d-6944-2d6a-b15d-561f403fc1d1/data/data9/current/BP-1552246360-172.17.0.2-1733934224410/current, will proceed with Du for space computation calculation, 
2024-12-11T16:24:00,667 WARN  [Thread-853 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-11T16:24:00,669 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb8bc4fe1095ae04 with lease ID 0x625a5ee7777b3bf1: Processing first storage report for DS-c9a8be59-e598-41e5-9034-32246b7a6ac3 from datanode DatanodeRegistration(127.0.0.1:40471, datanodeUuid=852f5a8b-0ced-4a59-b572-6ce7ea337674, infoPort=39651, infoSecurePort=0, ipcPort=42181, storageInfo=lv=-57;cid=testClusterID;nsid=1545217681;c=1733934224410)
2024-12-11T16:24:00,669 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb8bc4fe1095ae04 with lease ID 0x625a5ee7777b3bf1: from storage DS-c9a8be59-e598-41e5-9034-32246b7a6ac3 node DatanodeRegistration(127.0.0.1:40471, datanodeUuid=852f5a8b-0ced-4a59-b572-6ce7ea337674, infoPort=39651, infoSecurePort=0, ipcPort=42181, storageInfo=lv=-57;cid=testClusterID;nsid=1545217681;c=1733934224410), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-11T16:24:00,669 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb8bc4fe1095ae04 with lease ID 0x625a5ee7777b3bf1: Processing first storage report for DS-e0eb56a2-fe08-43ff-b420-1524702c59a0 from datanode DatanodeRegistration(127.0.0.1:40471, datanodeUuid=852f5a8b-0ced-4a59-b572-6ce7ea337674, infoPort=39651, infoSecurePort=0, ipcPort=42181, storageInfo=lv=-57;cid=testClusterID;nsid=1545217681;c=1733934224410)
2024-12-11T16:24:00,669 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb8bc4fe1095ae04 with lease ID 0x625a5ee7777b3bf1: from storage DS-e0eb56a2-fe08-43ff-b420-1524702c59a0 node DatanodeRegistration(127.0.0.1:40471, datanodeUuid=852f5a8b-0ced-4a59-b572-6ce7ea337674, infoPort=39651, infoSecurePort=0, ipcPort=42181, storageInfo=lv=-57;cid=testClusterID;nsid=1545217681;c=1733934224410), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-11T16:24:00,683 WARN  [ResponseProcessor for block BP-1552246360-172.17.0.2-1733934224410:blk_1073741834_1010 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1552246360-172.17.0.2-1733934224410:blk_1073741834_1010
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:00,684 WARN  [ResponseProcessor for block BP-1552246360-172.17.0.2-1733934224410:blk_1073741837_1013 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1552246360-172.17.0.2-1733934224410:blk_1073741837_1013
java.io.IOException: Bad response ERROR for BP-1552246360-172.17.0.2-1733934224410:blk_1073741837_1013 from datanode DatanodeInfoWithStorage[127.0.0.1:38801,DS-226c2206-3817-4f6d-b23c-9dac8d5d6b69,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:00,684 WARN  [DataStreamer for file /user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta block BP-1552246360-172.17.0.2-1733934224410:blk_1073741834_1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1552246360-172.17.0.2-1733934224410:blk_1073741834_1010 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38801,DS-226c2206-3817-4f6d-b23c-9dac8d5d6b69,DISK], DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38801,DS-226c2206-3817-4f6d-b23c-9dac8d5d6b69,DISK]) is bad.
2024-12-11T16:24:00,684 WARN  [ResponseProcessor for block BP-1552246360-172.17.0.2-1733934224410:blk_1073741830_1006 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1552246360-172.17.0.2-1733934224410:blk_1073741830_1006
java.io.IOException: Bad response ERROR for BP-1552246360-172.17.0.2-1733934224410:blk_1073741830_1006 from datanode DatanodeInfoWithStorage[127.0.0.1:38801,DS-226c2206-3817-4f6d-b23c-9dac8d5d6b69,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:00,684 WARN  [ResponseProcessor for block BP-1552246360-172.17.0.2-1733934224410:blk_1073741833_1009 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1552246360-172.17.0.2-1733934224410:blk_1073741833_1009
java.io.IOException: Bad response ERROR for BP-1552246360-172.17.0.2-1733934224410:blk_1073741833_1009 from datanode DatanodeInfoWithStorage[127.0.0.1:38801,DS-226c2206-3817-4f6d-b23c-9dac8d5d6b69,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:00,684 WARN  [DataStreamer for file /user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/MasterData/WALs/7446a65196a1,36693,1733934227006/7446a65196a1%2C36693%2C1733934227006.1733934227442 block BP-1552246360-172.17.0.2-1733934224410:blk_1073741830_1006 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1552246360-172.17.0.2-1733934224410:blk_1073741830_1006 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK], DatanodeInfoWithStorage[127.0.0.1:38801,DS-226c2206-3817-4f6d-b23c-9dac8d5d6b69,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:38801,DS-226c2206-3817-4f6d-b23c-9dac8d5d6b69,DISK]) is bad.
2024-12-11T16:24:00,684 WARN  [DataStreamer for file /user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064 block BP-1552246360-172.17.0.2-1733934224410:blk_1073741837_1013 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1552246360-172.17.0.2-1733934224410:blk_1073741837_1013 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK], DatanodeInfoWithStorage[127.0.0.1:38801,DS-226c2206-3817-4f6d-b23c-9dac8d5d6b69,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:38801,DS-226c2206-3817-4f6d-b23c-9dac8d5d6b69,DISK]) is bad.
2024-12-11T16:24:00,684 WARN  [PacketResponder: BP-1552246360-172.17.0.2-1733934224410:blk_1073741837_1013, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:38801] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T16:24:00,684 WARN  [PacketResponder: BP-1552246360-172.17.0.2-1733934224410:blk_1073741830_1006, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:38801] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T16:24:00,684 WARN  [DataStreamer for file /user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.1733934227980 block BP-1552246360-172.17.0.2-1733934224410:blk_1073741833_1009 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1552246360-172.17.0.2-1733934224410:blk_1073741833_1009 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK], DatanodeInfoWithStorage[127.0.0.1:38801,DS-226c2206-3817-4f6d-b23c-9dac8d5d6b69,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:38801,DS-226c2206-3817-4f6d-b23c-9dac8d5d6b69,DISK]) is bad.
2024-12-11T16:24:00,685 WARN  [PacketResponder: BP-1552246360-172.17.0.2-1733934224410:blk_1073741833_1009, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:38801] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T16:24:00,685 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_319672264_22 at /127.0.0.1:54402 [Receiving block BP-1552246360-172.17.0.2-1733934224410:blk_1073741837_1013] {}] datanode.DataXceiver(331): 127.0.0.1:44567:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:54402 dst: /127.0.0.1:44567
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T16:24:00,686 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-501712066_22 at /127.0.0.1:54376 [Receiving block BP-1552246360-172.17.0.2-1733934224410:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:44567:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:54376 dst: /127.0.0.1:44567
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T16:24:00,686 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5def8371{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T16:24:00,686 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-501712066_22 at /127.0.0.1:54374 [Receiving block BP-1552246360-172.17.0.2-1733934224410:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:44567:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:54374 dst: /127.0.0.1:44567
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T16:24:00,686 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_319672264_22 at /127.0.0.1:40524 [Receiving block BP-1552246360-172.17.0.2-1733934224410:blk_1073741837_1013] {}] datanode.DataXceiver(331): 127.0.0.1:38801:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:40524 dst: /127.0.0.1:38801
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T16:24:00,687 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@77f8647e{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-11T16:24:00,687 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-501712066_22 at /127.0.0.1:40490 [Receiving block BP-1552246360-172.17.0.2-1733934224410:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:38801:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:40490 dst: /127.0.0.1:38801
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T16:24:00,687 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-11T16:24:00,687 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-501712066_22 at /127.0.0.1:40474 [Receiving block BP-1552246360-172.17.0.2-1733934224410:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:38801:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:40474 dst: /127.0.0.1:38801
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T16:24:00,687 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-752322117_22 at /127.0.0.1:54348 [Receiving block BP-1552246360-172.17.0.2-1733934224410:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:44567:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:54348 dst: /127.0.0.1:44567
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T16:24:00,688 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@78ac40e9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-11T16:24:00,688 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@74afbef6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/hadoop.log.dir/,STOPPED}
2024-12-11T16:24:00,688 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-752322117_22 at /127.0.0.1:40464 [Receiving block BP-1552246360-172.17.0.2-1733934224410:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:38801:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:40464 dst: /127.0.0.1:38801
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T16:24:00,694 WARN  [BP-1552246360-172.17.0.2-1733934224410 heartbeating to localhost/127.0.0.1:34525 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-11T16:24:00,694 WARN  [BP-1552246360-172.17.0.2-1733934224410 heartbeating to localhost/127.0.0.1:34525 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1552246360-172.17.0.2-1733934224410 (Datanode Uuid 3250d228-4a4c-40f3-8d16-2b51b2a499ea) service to localhost/127.0.0.1:34525
2024-12-11T16:24:00,695 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/cluster_7052d23d-6944-2d6a-b15d-561f403fc1d1/data/data3/current/BP-1552246360-172.17.0.2-1733934224410 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T16:24:00,695 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/cluster_7052d23d-6944-2d6a-b15d-561f403fc1d1/data/data4/current/BP-1552246360-172.17.0.2-1733934224410 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T16:24:00,695 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-11T16:24:00,695 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-11T16:24:00,695 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-11T16:24:00,696 WARN  [DataStreamer for file /user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.1733934227980 block BP-1552246360-172.17.0.2-1733934224410:blk_1073741833_1009 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741833_1009
java.net.SocketException: Connection reset
	at sun.nio.ch.SocketChannelImpl.throwConnectionReset(SocketChannelImpl.java:394) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:426) ~[?:?]
	at org.apache.hadoop.net.SocketInputStream$Reader.performIO(SocketInputStream.java:57) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:118) ~[hadoop-common-3.4.1.jar:?]
	at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?]
	at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:527) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1931) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:00,697 WARN  [DataStreamer for file /user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064 block BP-1552246360-172.17.0.2-1733934224410:blk_1073741837_1013 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741837_1013
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:00,697 WARN  [DataStreamer for file /user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta block BP-1552246360-172.17.0.2-1733934224410:blk_1073741834_1010 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741834_1010
java.net.SocketException: Connection reset
	at sun.nio.ch.SocketChannelImpl.throwConnectionReset(SocketChannelImpl.java:394) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:426) ~[?:?]
	at org.apache.hadoop.net.SocketInputStream$Reader.performIO(SocketInputStream.java:57) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:118) ~[hadoop-common-3.4.1.jar:?]
	at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?]
	at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:527) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1931) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:00,697 ERROR [org.apache.hadoop.hdfs.server.datanode.DataXceiver@52476e67 {}] datanode.DataXceiver(331): 127.0.0.1:44567:DataXceiver error processing unknown operation  src: /127.0.0.1:37120 dst: /127.0.0.1:44567
java.io.IOException: Server closed.
	at org.apache.hadoop.hdfs.server.datanode.DataXceiverServer.addPeer(DataXceiverServer.java:334) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:232) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T16:24:00,698 WARN  [DataStreamer for file /user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/MasterData/WALs/7446a65196a1,36693,1733934227006/7446a65196a1%2C36693%2C1733934227006.1733934227442 block BP-1552246360-172.17.0.2-1733934224410:blk_1073741830_1006 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741830_1006
java.net.SocketException: Connection reset
	at sun.nio.ch.SocketChannelImpl.throwConnectionReset(SocketChannelImpl.java:394) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:426) ~[?:?]
	at org.apache.hadoop.net.SocketInputStream$Reader.performIO(SocketInputStream.java:57) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:118) ~[hadoop-common-3.4.1.jar:?]
	at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?]
	at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:527) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1931) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:00,699 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@390e4b46{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T16:24:00,699 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@78ec1620{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-11T16:24:00,699 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-11T16:24:00,700 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2c5fd7de{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-11T16:24:00,700 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5c11a3d2{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/hadoop.log.dir/,STOPPED}
2024-12-11T16:24:00,701 WARN  [BP-1552246360-172.17.0.2-1733934224410 heartbeating to localhost/127.0.0.1:34525 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-11T16:24:00,701 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-11T16:24:00,701 WARN  [BP-1552246360-172.17.0.2-1733934224410 heartbeating to localhost/127.0.0.1:34525 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1552246360-172.17.0.2-1733934224410 (Datanode Uuid 9bef2747-4727-42cc-97c6-d783b2288746) service to localhost/127.0.0.1:34525
2024-12-11T16:24:00,701 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-11T16:24:00,702 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/cluster_7052d23d-6944-2d6a-b15d-561f403fc1d1/data/data1/current/BP-1552246360-172.17.0.2-1733934224410 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T16:24:00,702 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/cluster_7052d23d-6944-2d6a-b15d-561f403fc1d1/data/data2/current/BP-1552246360-172.17.0.2-1733934224410 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T16:24:00,702 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-11T16:24:00,706 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRollOnDatanodeDeath', row='row0002', locateType=CURRENT is [region=TestLogRolling-testLogRollOnDatanodeDeath,,1733934228960.96a752a5036dfbc63de411b3f9c85ba7., hostname=7446a65196a1,45947,1733934227248, seqNum=2]
2024-12-11T16:24:00,708 ERROR [FSHLog-0-hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee-prefix:7446a65196a1,45947,1733934227248 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:00,708 WARN  [FSHLog-0-hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee-prefix:7446a65196a1,45947,1733934227248 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:00,708 DEBUG [regionserver/7446a65196a1:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 7446a65196a1%2C45947%2C1733934227248:(num 1733934227980) roll requested
2024-12-11T16:24:00,709 INFO  [regionserver/7446a65196a1:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7446a65196a1%2C45947%2C1733934227248.1733934240708
2024-12-11T16:24:00,716 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:24:00,716 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:24:00,716 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:24:00,716 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:24:00,716 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:24:00,716 INFO  [regionserver/7446a65196a1:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.1733934227980 with entries=1, filesize=455 B; new WAL /user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.1733934240708
2024-12-11T16:24:00,718 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:00,719 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:00,720 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils
2024-12-11T16:24:00,720 DEBUG [regionserver/7446a65196a1:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40753:40753),(127.0.0.1/127.0.0.1:32897:32897)]
2024-12-11T16:24:00,720 DEBUG [regionserver/7446a65196a1:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.1733934227980 is not closed yet, will try archiving it next time
2024-12-11T16:24:00,720 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease()
2024-12-11T16:24:00,720 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.1733934227980
2024-12-11T16:24:00,723 WARN  [IPC Server handler 1 on default port 34525 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.1733934227980 has not been closed. Lease recovery is in progress. RecoveryId = 1019 for block blk_1073741833_1009
2024-12-11T16:24:00,728 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.1733934227980 after 6ms
2024-12-11T16:24:00,923 INFO  [regionserver/7446a65196a1:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:01,424 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:02,720 INFO  [regionserver/7446a65196a1:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:02,722 INFO  [Time-limited test {}] wal.TestLogRolling(261): log.getCurrentFileName(): hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.1733934240708
2024-12-11T16:24:02,722 WARN  [ResponseProcessor for block BP-1552246360-172.17.0.2-1733934224410:blk_1073741838_1018 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1552246360-172.17.0.2-1733934224410:blk_1073741838_1018
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:02,723 WARN  [DataStreamer for file /user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.1733934240708 block BP-1552246360-172.17.0.2-1733934224410:blk_1073741838_1018 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1552246360-172.17.0.2-1733934224410:blk_1073741838_1018 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39649,DS-8e83c89e-91af-4855-9e85-7f0bb0de8fa8,DISK], DatanodeInfoWithStorage[127.0.0.1:35157,DS-d85b2b77-d19f-4302-be38-ce7ae4e72a21,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39649,DS-8e83c89e-91af-4855-9e85-7f0bb0de8fa8,DISK]) is bad.
2024-12-11T16:24:02,724 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-501712066_22 at /127.0.0.1:35298 [Receiving block BP-1552246360-172.17.0.2-1733934224410:blk_1073741838_1018] {}] datanode.DataXceiver(331): 127.0.0.1:39649:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:35298 dst: /127.0.0.1:39649
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T16:24:02,725 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-501712066_22 at /127.0.0.1:57030 [Receiving block BP-1552246360-172.17.0.2-1733934224410:blk_1073741838_1018] {}] datanode.DataXceiver(331): 127.0.0.1:35157:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:57030 dst: /127.0.0.1:35157
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T16:24:02,729 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5424ebd1{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T16:24:02,729 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@70ae18b0{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-11T16:24:02,729 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-11T16:24:02,730 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@41b3c9b2{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-11T16:24:02,730 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1f5f345{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/hadoop.log.dir/,STOPPED}
2024-12-11T16:24:02,742 WARN  [BP-1552246360-172.17.0.2-1733934224410 heartbeating to localhost/127.0.0.1:34525 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-11T16:24:02,742 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-11T16:24:02,742 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-11T16:24:02,742 WARN  [BP-1552246360-172.17.0.2-1733934224410 heartbeating to localhost/127.0.0.1:34525 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1552246360-172.17.0.2-1733934224410 (Datanode Uuid de133113-d741-4d77-878b-bfdb00c87c5e) service to localhost/127.0.0.1:34525
2024-12-11T16:24:02,743 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/cluster_7052d23d-6944-2d6a-b15d-561f403fc1d1/data/data5/current/BP-1552246360-172.17.0.2-1733934224410 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T16:24:02,743 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/cluster_7052d23d-6944-2d6a-b15d-561f403fc1d1/data/data6/current/BP-1552246360-172.17.0.2-1733934224410 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T16:24:02,743 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-11T16:24:02,924 INFO  [regionserver/7446a65196a1:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:03,424 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:04,721 WARN  [regionserver/7446a65196a1:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:35157,DS-d85b2b77-d19f-4302-be38-ce7ae4e72a21,DISK]]
2024-12-11T16:24:04,721 INFO  [regionserver/7446a65196a1:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:04,721 DEBUG [regionserver/7446a65196a1:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 7446a65196a1%2C45947%2C1733934227248:(num 1733934240708) roll requested
2024-12-11T16:24:04,722 INFO  [regionserver/7446a65196a1:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7446a65196a1%2C45947%2C1733934227248.1733934244722
2024-12-11T16:24:04,732 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.1733934227980 after 4012ms
2024-12-11T16:24:04,749 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2024-12-11T16:24:04,924 INFO  [regionserver/7446a65196a1:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:05,020 WARN  [Thread-911 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741839_1021
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:44567
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:05,020 WARN  [Thread-911 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1552246360-172.17.0.2-1733934224410:blk_1073741839_1021 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35157,DS-d85b2b77-d19f-4302-be38-ce7ae4e72a21,DISK], DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK]) is bad.
2024-12-11T16:24:05,019 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-501712066_22 at /127.0.0.1:33016 [Receiving block BP-1552246360-172.17.0.2-1733934224410:blk_1073741839_1021] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/cluster_7052d23d-6944-2d6a-b15d-561f403fc1d1/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/cluster_7052d23d-6944-2d6a-b15d-561f403fc1d1/data/data8]'}, localName='127.0.0.1:35157', datanodeUuid='690ba6fc-5c14-4f6a-9c83-e257a13f29f3', xmitsInProgress=0}:Exception transferring block BP-1552246360-172.17.0.2-1733934224410:blk_1073741839_1021 to mirror 127.0.0.1:44567
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T16:24:05,020 WARN  [Thread-911 {}] hdfs.DataStreamer(1850): Abandoning BP-1552246360-172.17.0.2-1733934224410:blk_1073741839_1021
2024-12-11T16:24:05,020 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-501712066_22 at /127.0.0.1:33016 [Receiving block BP-1552246360-172.17.0.2-1733934224410:blk_1073741839_1021] {}] datanode.BlockReceiver(316): Block 1073741839 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-12-11T16:24:05,020 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-501712066_22 at /127.0.0.1:33016 [Receiving block BP-1552246360-172.17.0.2-1733934224410:blk_1073741839_1021] {}] datanode.DataXceiver(331): 127.0.0.1:35157:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:33016 dst: /127.0.0.1:35157
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T16:24:05,023 WARN  [Thread-911 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK]
2024-12-11T16:24:05,053 INFO  [regionserver/7446a65196a1:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 326 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:35157,DS-d85b2b77-d19f-4302-be38-ce7ae4e72a21,DISK]]
2024-12-11T16:24:05,054 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:24:05,054 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:24:05,054 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:24:05,054 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:24:05,054 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:24:05,055 INFO  [regionserver/7446a65196a1:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.1733934240708 with entries=3, filesize=3.51 KB; new WAL /user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.1733934244722
2024-12-11T16:24:05,066 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35157 is added to blk_1073741838_1020 (size=3600)
2024-12-11T16:24:05,069 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.1733934227980 is not closed yet, will try archiving it next time
2024-12-11T16:24:05,091 DEBUG [regionserver/7446a65196a1:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:32897:32897),(127.0.0.1/127.0.0.1:39651:39651)]
2024-12-11T16:24:05,092 DEBUG [regionserver/7446a65196a1:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.1733934227980 is not closed yet, will try archiving it next time
2024-12-11T16:24:05,424 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:06,549 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@67feaf7b[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:35157, datanodeUuid=690ba6fc-5c14-4f6a-9c83-e257a13f29f3, infoPort=32897, infoSecurePort=0, ipcPort=42115, storageInfo=lv=-57;cid=testClusterID;nsid=1545217681;c=1733934224410):Failed to transfer BP-1552246360-172.17.0.2-1733934224410:blk_1073741838_1020 to 127.0.0.1:44567 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T16:24:06,753 WARN  [ResponseProcessor for block BP-1552246360-172.17.0.2-1733934224410:blk_1073741840_1022 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1552246360-172.17.0.2-1733934224410:blk_1073741840_1022
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:06,754 WARN  [DataStreamer for file /user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.1733934244722 block BP-1552246360-172.17.0.2-1733934224410:blk_1073741840_1022 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1552246360-172.17.0.2-1733934224410:blk_1073741840_1022 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35157,DS-d85b2b77-d19f-4302-be38-ce7ae4e72a21,DISK], DatanodeInfoWithStorage[127.0.0.1:40471,DS-c9a8be59-e598-41e5-9034-32246b7a6ac3,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35157,DS-d85b2b77-d19f-4302-be38-ce7ae4e72a21,DISK]) is bad.
2024-12-11T16:24:06,754 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-501712066_22 at /127.0.0.1:33022 [Receiving block BP-1552246360-172.17.0.2-1733934224410:blk_1073741840_1022] {}] datanode.DataXceiver(331): 127.0.0.1:35157:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:33022 dst: /127.0.0.1:35157
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T16:24:06,754 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-501712066_22 at /127.0.0.1:33742 [Receiving block BP-1552246360-172.17.0.2-1733934224410:blk_1073741840_1022] {}] datanode.DataXceiver(331): 127.0.0.1:40471:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:33742 dst: /127.0.0.1:40471
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T16:24:06,756 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@768a1767{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T16:24:06,756 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@44bcc293{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-11T16:24:06,756 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-11T16:24:06,756 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2d7d08f9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-11T16:24:06,756 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@20c0741c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/hadoop.log.dir/,STOPPED}
2024-12-11T16:24:06,758 WARN  [BP-1552246360-172.17.0.2-1733934224410 heartbeating to localhost/127.0.0.1:34525 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-11T16:24:06,758 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-11T16:24:06,758 WARN  [BP-1552246360-172.17.0.2-1733934224410 heartbeating to localhost/127.0.0.1:34525 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1552246360-172.17.0.2-1733934224410 (Datanode Uuid 690ba6fc-5c14-4f6a-9c83-e257a13f29f3) service to localhost/127.0.0.1:34525
2024-12-11T16:24:06,758 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-11T16:24:06,759 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/cluster_7052d23d-6944-2d6a-b15d-561f403fc1d1/data/data7/current/BP-1552246360-172.17.0.2-1733934224410 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T16:24:06,759 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/cluster_7052d23d-6944-2d6a-b15d-561f403fc1d1/data/data8/current/BP-1552246360-172.17.0.2-1733934224410 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T16:24:06,759 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-11T16:24:06,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45947 {}] regionserver.HRegion(8855): Flush requested on 96a752a5036dfbc63de411b3f9c85ba7
2024-12-11T16:24:06,779 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96a752a5036dfbc63de411b3f9c85ba7 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-11T16:24:06,799 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/.tmp/info/3afb4e2b0a8043cea9a75cbf8e8db744 is 1080, key is row0002/info:/1733934242744/Put/seqid=0
2024-12-11T16:24:06,802 WARN  [Thread-923 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741841_1024
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:06,802 WARN  [Thread-923 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1552246360-172.17.0.2-1733934224410:blk_1073741841_1024 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39649,DS-8e83c89e-91af-4855-9e85-7f0bb0de8fa8,DISK], DatanodeInfoWithStorage[127.0.0.1:40471,DS-c9a8be59-e598-41e5-9034-32246b7a6ac3,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39649,DS-8e83c89e-91af-4855-9e85-7f0bb0de8fa8,DISK]) is bad.
2024-12-11T16:24:06,802 WARN  [Thread-923 {}] hdfs.DataStreamer(1850): Abandoning BP-1552246360-172.17.0.2-1733934224410:blk_1073741841_1024
2024-12-11T16:24:06,803 WARN  [Thread-923 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39649,DS-8e83c89e-91af-4855-9e85-7f0bb0de8fa8,DISK]
2024-12-11T16:24:06,805 WARN  [Thread-923 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741842_1025
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:06,805 WARN  [Thread-923 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1552246360-172.17.0.2-1733934224410:blk_1073741842_1025 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38801,DS-226c2206-3817-4f6d-b23c-9dac8d5d6b69,DISK], DatanodeInfoWithStorage[127.0.0.1:35157,DS-d85b2b77-d19f-4302-be38-ce7ae4e72a21,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38801,DS-226c2206-3817-4f6d-b23c-9dac8d5d6b69,DISK]) is bad.
2024-12-11T16:24:06,805 WARN  [Thread-923 {}] hdfs.DataStreamer(1850): Abandoning BP-1552246360-172.17.0.2-1733934224410:blk_1073741842_1025
2024-12-11T16:24:06,806 WARN  [Thread-923 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38801,DS-226c2206-3817-4f6d-b23c-9dac8d5d6b69,DISK]
2024-12-11T16:24:06,809 WARN  [Thread-923 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741843_1026
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:06,809 WARN  [Thread-923 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1552246360-172.17.0.2-1733934224410:blk_1073741843_1026 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35157,DS-d85b2b77-d19f-4302-be38-ce7ae4e72a21,DISK], DatanodeInfoWithStorage[127.0.0.1:40471,DS-c9a8be59-e598-41e5-9034-32246b7a6ac3,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35157,DS-d85b2b77-d19f-4302-be38-ce7ae4e72a21,DISK]) is bad.
2024-12-11T16:24:06,809 WARN  [Thread-923 {}] hdfs.DataStreamer(1850): Abandoning BP-1552246360-172.17.0.2-1733934224410:blk_1073741843_1026
2024-12-11T16:24:06,810 WARN  [Thread-923 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35157,DS-d85b2b77-d19f-4302-be38-ce7ae4e72a21,DISK]
2024-12-11T16:24:06,812 WARN  [Thread-923 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741844_1027
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:06,812 WARN  [Thread-923 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1552246360-172.17.0.2-1733934224410:blk_1073741844_1027 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK], DatanodeInfoWithStorage[127.0.0.1:40471,DS-c9a8be59-e598-41e5-9034-32246b7a6ac3,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK]) is bad.
2024-12-11T16:24:06,812 WARN  [Thread-923 {}] hdfs.DataStreamer(1850): Abandoning BP-1552246360-172.17.0.2-1733934224410:blk_1073741844_1027
2024-12-11T16:24:06,813 WARN  [Thread-923 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK]
2024-12-11T16:24:06,814 WARN  [IPC Server handler 2 on default port 34525 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-11T16:24:06,814 WARN  [IPC Server handler 2 on default port 34525 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-11T16:24:06,814 WARN  [IPC Server handler 2 on default port 34525 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-11T16:24:06,819 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40471 is added to blk_1073741845_1028 (size=10347)
2024-12-11T16:24:06,820 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/.tmp/info/3afb4e2b0a8043cea9a75cbf8e8db744
2024-12-11T16:24:06,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/.tmp/info/3afb4e2b0a8043cea9a75cbf8e8db744 as hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/info/3afb4e2b0a8043cea9a75cbf8e8db744
2024-12-11T16:24:06,843 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/info/3afb4e2b0a8043cea9a75cbf8e8db744, entries=5, sequenceid=11, filesize=10.1 K
2024-12-11T16:24:06,844 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=7.35 KB/7531 for 96a752a5036dfbc63de411b3f9c85ba7 in 65ms, sequenceid=11, compaction requested=false
2024-12-11T16:24:06,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96a752a5036dfbc63de411b3f9c85ba7:

2024-12-11T16:24:06,925 INFO  [regionserver/7446a65196a1:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:07,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45947 {}] regionserver.HRegion(8855): Flush requested on 96a752a5036dfbc63de411b3f9c85ba7
2024-12-11T16:24:07,002 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96a752a5036dfbc63de411b3f9c85ba7 1/1 column families, dataSize=8.40 KB heapSize=9.25 KB
2024-12-11T16:24:07,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/.tmp/info/9120724ff6334105b29217fc805dd912 is 1080, key is row0007/info:/1733934246781/Put/seqid=0
2024-12-11T16:24:07,008 WARN  [Thread-928 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741846_1029
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:07,009 WARN  [Thread-928 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1552246360-172.17.0.2-1733934224410:blk_1073741846_1029 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39649,DS-8e83c89e-91af-4855-9e85-7f0bb0de8fa8,DISK], DatanodeInfoWithStorage[127.0.0.1:40471,DS-c9a8be59-e598-41e5-9034-32246b7a6ac3,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39649,DS-8e83c89e-91af-4855-9e85-7f0bb0de8fa8,DISK]) is bad.
2024-12-11T16:24:07,009 WARN  [Thread-928 {}] hdfs.DataStreamer(1850): Abandoning BP-1552246360-172.17.0.2-1733934224410:blk_1073741846_1029
2024-12-11T16:24:07,009 WARN  [Thread-928 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39649,DS-8e83c89e-91af-4855-9e85-7f0bb0de8fa8,DISK]
2024-12-11T16:24:07,011 WARN  [Thread-928 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741847_1030
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:07,011 WARN  [Thread-928 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1552246360-172.17.0.2-1733934224410:blk_1073741847_1030 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35157,DS-d85b2b77-d19f-4302-be38-ce7ae4e72a21,DISK], DatanodeInfoWithStorage[127.0.0.1:40471,DS-c9a8be59-e598-41e5-9034-32246b7a6ac3,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35157,DS-d85b2b77-d19f-4302-be38-ce7ae4e72a21,DISK]) is bad.
2024-12-11T16:24:07,011 WARN  [Thread-928 {}] hdfs.DataStreamer(1850): Abandoning BP-1552246360-172.17.0.2-1733934224410:blk_1073741847_1030
2024-12-11T16:24:07,011 WARN  [Thread-928 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35157,DS-d85b2b77-d19f-4302-be38-ce7ae4e72a21,DISK]
2024-12-11T16:24:07,013 WARN  [Thread-928 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741848_1031
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:07,013 WARN  [Thread-928 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1552246360-172.17.0.2-1733934224410:blk_1073741848_1031 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK], DatanodeInfoWithStorage[127.0.0.1:38801,DS-226c2206-3817-4f6d-b23c-9dac8d5d6b69,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK]) is bad.
2024-12-11T16:24:07,013 WARN  [Thread-928 {}] hdfs.DataStreamer(1850): Abandoning BP-1552246360-172.17.0.2-1733934224410:blk_1073741848_1031
2024-12-11T16:24:07,014 WARN  [Thread-928 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK]
2024-12-11T16:24:07,016 WARN  [Thread-928 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741849_1032
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:38801
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:07,016 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-501712066_22 at /127.0.0.1:33788 [Receiving block BP-1552246360-172.17.0.2-1733934224410:blk_1073741849_1032] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/cluster_7052d23d-6944-2d6a-b15d-561f403fc1d1/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/cluster_7052d23d-6944-2d6a-b15d-561f403fc1d1/data/data10]'}, localName='127.0.0.1:40471', datanodeUuid='852f5a8b-0ced-4a59-b572-6ce7ea337674', xmitsInProgress=0}:Exception transferring block BP-1552246360-172.17.0.2-1733934224410:blk_1073741849_1032 to mirror 127.0.0.1:38801
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T16:24:07,016 WARN  [Thread-928 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1552246360-172.17.0.2-1733934224410:blk_1073741849_1032 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40471,DS-c9a8be59-e598-41e5-9034-32246b7a6ac3,DISK], DatanodeInfoWithStorage[127.0.0.1:38801,DS-226c2206-3817-4f6d-b23c-9dac8d5d6b69,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:38801,DS-226c2206-3817-4f6d-b23c-9dac8d5d6b69,DISK]) is bad.
2024-12-11T16:24:07,016 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-501712066_22 at /127.0.0.1:33788 [Receiving block BP-1552246360-172.17.0.2-1733934224410:blk_1073741849_1032] {}] datanode.BlockReceiver(316): Block 1073741849 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-11T16:24:07,016 WARN  [Thread-928 {}] hdfs.DataStreamer(1850): Abandoning BP-1552246360-172.17.0.2-1733934224410:blk_1073741849_1032
2024-12-11T16:24:07,016 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-501712066_22 at /127.0.0.1:33788 [Receiving block BP-1552246360-172.17.0.2-1733934224410:blk_1073741849_1032] {}] datanode.DataXceiver(331): 127.0.0.1:40471:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:33788 dst: /127.0.0.1:40471
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T16:24:07,017 WARN  [Thread-928 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38801,DS-226c2206-3817-4f6d-b23c-9dac8d5d6b69,DISK]
2024-12-11T16:24:07,018 WARN  [IPC Server handler 0 on default port 34525 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-11T16:24:07,018 WARN  [IPC Server handler 0 on default port 34525 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-11T16:24:07,018 WARN  [IPC Server handler 0 on default port 34525 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-11T16:24:07,021 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40471 is added to blk_1073741850_1033 (size=12506)
2024-12-11T16:24:07,092 WARN  [regionserver/7446a65196a1:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40471,DS-c9a8be59-e598-41e5-9034-32246b7a6ac3,DISK]]
2024-12-11T16:24:07,092 INFO  [regionserver/7446a65196a1:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:07,092 DEBUG [regionserver/7446a65196a1:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 7446a65196a1%2C45947%2C1733934227248:(num 1733934244722) roll requested
2024-12-11T16:24:07,093 INFO  [regionserver/7446a65196a1:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7446a65196a1%2C45947%2C1733934227248.1733934247093
2024-12-11T16:24:07,097 WARN  [Thread-933 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741851_1034
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:35157
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:07,097 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-501712066_22 at /127.0.0.1:33808 [Receiving block BP-1552246360-172.17.0.2-1733934224410:blk_1073741851_1034] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/cluster_7052d23d-6944-2d6a-b15d-561f403fc1d1/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/cluster_7052d23d-6944-2d6a-b15d-561f403fc1d1/data/data10]'}, localName='127.0.0.1:40471', datanodeUuid='852f5a8b-0ced-4a59-b572-6ce7ea337674', xmitsInProgress=0}:Exception transferring block BP-1552246360-172.17.0.2-1733934224410:blk_1073741851_1034 to mirror 127.0.0.1:35157
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T16:24:07,098 WARN  [Thread-933 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1552246360-172.17.0.2-1733934224410:blk_1073741851_1034 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40471,DS-c9a8be59-e598-41e5-9034-32246b7a6ac3,DISK], DatanodeInfoWithStorage[127.0.0.1:35157,DS-d85b2b77-d19f-4302-be38-ce7ae4e72a21,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:35157,DS-d85b2b77-d19f-4302-be38-ce7ae4e72a21,DISK]) is bad.
2024-12-11T16:24:07,098 WARN  [Thread-933 {}] hdfs.DataStreamer(1850): Abandoning BP-1552246360-172.17.0.2-1733934224410:blk_1073741851_1034
2024-12-11T16:24:07,098 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-501712066_22 at /127.0.0.1:33808 [Receiving block BP-1552246360-172.17.0.2-1733934224410:blk_1073741851_1034] {}] datanode.BlockReceiver(316): Block 1073741851 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-12-11T16:24:07,098 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-501712066_22 at /127.0.0.1:33808 [Receiving block BP-1552246360-172.17.0.2-1733934224410:blk_1073741851_1034] {}] datanode.DataXceiver(331): 127.0.0.1:40471:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:33808 dst: /127.0.0.1:40471
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T16:24:07,098 WARN  [Thread-933 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35157,DS-d85b2b77-d19f-4302-be38-ce7ae4e72a21,DISK]
2024-12-11T16:24:07,100 WARN  [Thread-933 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741852_1035
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:07,100 WARN  [Thread-933 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1552246360-172.17.0.2-1733934224410:blk_1073741852_1035 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39649,DS-8e83c89e-91af-4855-9e85-7f0bb0de8fa8,DISK], DatanodeInfoWithStorage[127.0.0.1:38801,DS-226c2206-3817-4f6d-b23c-9dac8d5d6b69,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39649,DS-8e83c89e-91af-4855-9e85-7f0bb0de8fa8,DISK]) is bad.
2024-12-11T16:24:07,100 WARN  [Thread-933 {}] hdfs.DataStreamer(1850): Abandoning BP-1552246360-172.17.0.2-1733934224410:blk_1073741852_1035
2024-12-11T16:24:07,101 WARN  [Thread-933 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39649,DS-8e83c89e-91af-4855-9e85-7f0bb0de8fa8,DISK]
2024-12-11T16:24:07,102 WARN  [Thread-933 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741853_1036
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:07,102 WARN  [Thread-933 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1552246360-172.17.0.2-1733934224410:blk_1073741853_1036 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK], DatanodeInfoWithStorage[127.0.0.1:38801,DS-226c2206-3817-4f6d-b23c-9dac8d5d6b69,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK]) is bad.
2024-12-11T16:24:07,102 WARN  [Thread-933 {}] hdfs.DataStreamer(1850): Abandoning BP-1552246360-172.17.0.2-1733934224410:blk_1073741853_1036
2024-12-11T16:24:07,103 WARN  [Thread-933 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK]
2024-12-11T16:24:07,105 WARN  [Thread-933 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741854_1037
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:38801
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:07,105 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-501712066_22 at /127.0.0.1:33818 [Receiving block BP-1552246360-172.17.0.2-1733934224410:blk_1073741854_1037] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/cluster_7052d23d-6944-2d6a-b15d-561f403fc1d1/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/cluster_7052d23d-6944-2d6a-b15d-561f403fc1d1/data/data10]'}, localName='127.0.0.1:40471', datanodeUuid='852f5a8b-0ced-4a59-b572-6ce7ea337674', xmitsInProgress=0}:Exception transferring block BP-1552246360-172.17.0.2-1733934224410:blk_1073741854_1037 to mirror 127.0.0.1:38801
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T16:24:07,105 WARN  [Thread-933 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1552246360-172.17.0.2-1733934224410:blk_1073741854_1037 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40471,DS-c9a8be59-e598-41e5-9034-32246b7a6ac3,DISK], DatanodeInfoWithStorage[127.0.0.1:38801,DS-226c2206-3817-4f6d-b23c-9dac8d5d6b69,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:38801,DS-226c2206-3817-4f6d-b23c-9dac8d5d6b69,DISK]) is bad.
2024-12-11T16:24:07,105 WARN  [Thread-933 {}] hdfs.DataStreamer(1850): Abandoning BP-1552246360-172.17.0.2-1733934224410:blk_1073741854_1037
2024-12-11T16:24:07,105 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-501712066_22 at /127.0.0.1:33818 [Receiving block BP-1552246360-172.17.0.2-1733934224410:blk_1073741854_1037] {}] datanode.BlockReceiver(316): Block 1073741854 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-12-11T16:24:07,105 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-501712066_22 at /127.0.0.1:33818 [Receiving block BP-1552246360-172.17.0.2-1733934224410:blk_1073741854_1037] {}] datanode.DataXceiver(331): 127.0.0.1:40471:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:33818 dst: /127.0.0.1:40471
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T16:24:07,106 WARN  [Thread-933 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38801,DS-226c2206-3817-4f6d-b23c-9dac8d5d6b69,DISK]
2024-12-11T16:24:07,106 WARN  [IPC Server handler 2 on default port 34525 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-11T16:24:07,107 WARN  [IPC Server handler 2 on default port 34525 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-11T16:24:07,107 WARN  [IPC Server handler 2 on default port 34525 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-11T16:24:07,109 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:24:07,109 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:24:07,109 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:24:07,109 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:24:07,110 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:24:07,110 INFO  [regionserver/7446a65196a1:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.1733934244722 with entries=15, filesize=14.76 KB; new WAL /user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.1733934247093
2024-12-11T16:24:07,110 DEBUG [regionserver/7446a65196a1:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39651:39651)]
2024-12-11T16:24:07,111 DEBUG [regionserver/7446a65196a1:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.1733934227980 is not closed yet, will try archiving it next time
2024-12-11T16:24:07,111 DEBUG [regionserver/7446a65196a1:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.1733934244722 is not closed yet, will try archiving it next time
2024-12-11T16:24:07,111 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.1733934240708 to hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/oldWALs/7446a65196a1%2C45947%2C1733934227248.1733934240708
2024-12-11T16:24:07,112 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40471 is added to blk_1073741840_1023 (size=15122)
2024-12-11T16:24:07,424 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.40 KB at sequenceid=22 (bloomFilter=true), to=hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/.tmp/info/9120724ff6334105b29217fc805dd912
2024-12-11T16:24:07,425 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:07,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/.tmp/info/9120724ff6334105b29217fc805dd912 as hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/info/9120724ff6334105b29217fc805dd912
2024-12-11T16:24:07,443 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/info/9120724ff6334105b29217fc805dd912, entries=7, sequenceid=22, filesize=12.2 K
2024-12-11T16:24:07,444 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~8.40 KB/8606, heapSize ~9.23 KB/9456, currentSize=2.10 KB/2150 for 96a752a5036dfbc63de411b3f9c85ba7 in 442ms, sequenceid=22, compaction requested=false
2024-12-11T16:24:07,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96a752a5036dfbc63de411b3f9c85ba7:

2024-12-11T16:24:07,445 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=22.3 K, sizeToCheck=16.0 K
2024-12-11T16:24:07,445 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-11T16:24:07,445 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/info/9120724ff6334105b29217fc805dd912 because midkey is the same as first or last row
2024-12-11T16:24:07,513 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.1733934227980 is not closed yet, will try archiving it next time
2024-12-11T16:24:07,514 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.1733934244722 to hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/oldWALs/7446a65196a1%2C45947%2C1733934227248.1733934244722
2024-12-11T16:24:08,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45947 {}] regionserver.HRegion(8855): Flush requested on 96a752a5036dfbc63de411b3f9c85ba7
2024-12-11T16:24:08,423 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96a752a5036dfbc63de411b3f9c85ba7 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-12-11T16:24:08,428 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/.tmp/info/dfbc5851194d473991cff9e3fe8e2427 is 1079, key is tmprow/info:/1733934248422/Put/seqid=0
2024-12-11T16:24:08,432 WARN  [Thread-939 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741856_1039
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:44567
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:08,431 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-501712066_22 at /127.0.0.1:33836 [Receiving block BP-1552246360-172.17.0.2-1733934224410:blk_1073741856_1039] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/cluster_7052d23d-6944-2d6a-b15d-561f403fc1d1/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/cluster_7052d23d-6944-2d6a-b15d-561f403fc1d1/data/data10]'}, localName='127.0.0.1:40471', datanodeUuid='852f5a8b-0ced-4a59-b572-6ce7ea337674', xmitsInProgress=0}:Exception transferring block BP-1552246360-172.17.0.2-1733934224410:blk_1073741856_1039 to mirror 127.0.0.1:44567
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T16:24:08,432 WARN  [Thread-939 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1552246360-172.17.0.2-1733934224410:blk_1073741856_1039 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40471,DS-c9a8be59-e598-41e5-9034-32246b7a6ac3,DISK], DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK]) is bad.
2024-12-11T16:24:08,432 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-501712066_22 at /127.0.0.1:33836 [Receiving block BP-1552246360-172.17.0.2-1733934224410:blk_1073741856_1039] {}] datanode.BlockReceiver(316): Block 1073741856 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-11T16:24:08,432 WARN  [Thread-939 {}] hdfs.DataStreamer(1850): Abandoning BP-1552246360-172.17.0.2-1733934224410:blk_1073741856_1039
2024-12-11T16:24:08,432 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-501712066_22 at /127.0.0.1:33836 [Receiving block BP-1552246360-172.17.0.2-1733934224410:blk_1073741856_1039] {}] datanode.DataXceiver(331): 127.0.0.1:40471:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:33836 dst: /127.0.0.1:40471
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T16:24:08,433 WARN  [Thread-939 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK]
2024-12-11T16:24:08,434 WARN  [Thread-939 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741857_1040
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:08,434 WARN  [Thread-939 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1552246360-172.17.0.2-1733934224410:blk_1073741857_1040 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39649,DS-8e83c89e-91af-4855-9e85-7f0bb0de8fa8,DISK], DatanodeInfoWithStorage[127.0.0.1:38801,DS-226c2206-3817-4f6d-b23c-9dac8d5d6b69,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39649,DS-8e83c89e-91af-4855-9e85-7f0bb0de8fa8,DISK]) is bad.
2024-12-11T16:24:08,434 WARN  [Thread-939 {}] hdfs.DataStreamer(1850): Abandoning BP-1552246360-172.17.0.2-1733934224410:blk_1073741857_1040
2024-12-11T16:24:08,435 WARN  [Thread-939 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39649,DS-8e83c89e-91af-4855-9e85-7f0bb0de8fa8,DISK]
2024-12-11T16:24:08,436 WARN  [Thread-939 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741858_1041
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:08,436 WARN  [Thread-939 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1552246360-172.17.0.2-1733934224410:blk_1073741858_1041 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35157,DS-d85b2b77-d19f-4302-be38-ce7ae4e72a21,DISK], DatanodeInfoWithStorage[127.0.0.1:38801,DS-226c2206-3817-4f6d-b23c-9dac8d5d6b69,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35157,DS-d85b2b77-d19f-4302-be38-ce7ae4e72a21,DISK]) is bad.
2024-12-11T16:24:08,436 WARN  [Thread-939 {}] hdfs.DataStreamer(1850): Abandoning BP-1552246360-172.17.0.2-1733934224410:blk_1073741858_1041
2024-12-11T16:24:08,437 WARN  [Thread-939 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35157,DS-d85b2b77-d19f-4302-be38-ce7ae4e72a21,DISK]
2024-12-11T16:24:08,438 WARN  [Thread-939 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741859_1042
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:08,438 WARN  [Thread-939 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1552246360-172.17.0.2-1733934224410:blk_1073741859_1042 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38801,DS-226c2206-3817-4f6d-b23c-9dac8d5d6b69,DISK], DatanodeInfoWithStorage[127.0.0.1:40471,DS-c9a8be59-e598-41e5-9034-32246b7a6ac3,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38801,DS-226c2206-3817-4f6d-b23c-9dac8d5d6b69,DISK]) is bad.
2024-12-11T16:24:08,438 WARN  [Thread-939 {}] hdfs.DataStreamer(1850): Abandoning BP-1552246360-172.17.0.2-1733934224410:blk_1073741859_1042
2024-12-11T16:24:08,439 WARN  [Thread-939 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38801,DS-226c2206-3817-4f6d-b23c-9dac8d5d6b69,DISK]
2024-12-11T16:24:08,440 WARN  [IPC Server handler 0 on default port 34525 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-11T16:24:08,440 WARN  [IPC Server handler 0 on default port 34525 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-11T16:24:08,440 WARN  [IPC Server handler 0 on default port 34525 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-11T16:24:08,443 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40471 is added to blk_1073741860_1043 (size=6027)
2024-12-11T16:24:08,845 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/.tmp/info/dfbc5851194d473991cff9e3fe8e2427
2024-12-11T16:24:08,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/.tmp/info/dfbc5851194d473991cff9e3fe8e2427 as hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/info/dfbc5851194d473991cff9e3fe8e2427
2024-12-11T16:24:08,863 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/info/dfbc5851194d473991cff9e3fe8e2427, entries=1, sequenceid=32, filesize=5.9 K
2024-12-11T16:24:08,865 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for 96a752a5036dfbc63de411b3f9c85ba7 in 441ms, sequenceid=32, compaction requested=true
2024-12-11T16:24:08,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96a752a5036dfbc63de411b3f9c85ba7:

2024-12-11T16:24:08,865 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=28.2 K, sizeToCheck=16.0 K
2024-12-11T16:24:08,865 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-11T16:24:08,865 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/info/9120724ff6334105b29217fc805dd912 because midkey is the same as first or last row
2024-12-11T16:24:08,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96a752a5036dfbc63de411b3f9c85ba7:info, priority=-2147483648, current under compaction store size is 1
2024-12-11T16:24:08,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-11T16:24:08,866 DEBUG [RS:0;7446a65196a1:45947-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-11T16:24:08,867 DEBUG [RS:0;7446a65196a1:45947-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 28880 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-11T16:24:08,867 DEBUG [RS:0;7446a65196a1:45947-shortCompactions-0 {}] regionserver.HStore(1541): 96a752a5036dfbc63de411b3f9c85ba7/info is initiating minor compaction (all files)
2024-12-11T16:24:08,868 INFO  [RS:0;7446a65196a1:45947-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96a752a5036dfbc63de411b3f9c85ba7/info in TestLogRolling-testLogRollOnDatanodeDeath,,1733934228960.96a752a5036dfbc63de411b3f9c85ba7.
2024-12-11T16:24:08,868 INFO  [RS:0;7446a65196a1:45947-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/info/3afb4e2b0a8043cea9a75cbf8e8db744, hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/info/9120724ff6334105b29217fc805dd912, hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/info/dfbc5851194d473991cff9e3fe8e2427] into tmpdir=hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/.tmp, totalSize=28.2 K
2024-12-11T16:24:08,868 DEBUG [RS:0;7446a65196a1:45947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3afb4e2b0a8043cea9a75cbf8e8db744, keycount=5, bloomtype=ROW, size=10.1 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733934242744
2024-12-11T16:24:08,869 DEBUG [RS:0;7446a65196a1:45947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9120724ff6334105b29217fc805dd912, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=22, earliestPutTs=1733934246781
2024-12-11T16:24:08,869 DEBUG [RS:0;7446a65196a1:45947-shortCompactions-0 {}] compactions.Compactor(225): Compacting dfbc5851194d473991cff9e3fe8e2427, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=32, earliestPutTs=1733934248422
2024-12-11T16:24:08,882 INFO  [RS:0;7446a65196a1:45947-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96a752a5036dfbc63de411b3f9c85ba7#info#compaction#21 average throughput is 6.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-11T16:24:08,882 DEBUG [RS:0;7446a65196a1:45947-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/.tmp/info/fca5937ef3da4f0cb79cd7baff0186e5 is 1080, key is row0002/info:/1733934242744/Put/seqid=0
2024-12-11T16:24:08,884 WARN  [Thread-946 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741861_1044
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:08,884 WARN  [Thread-946 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1552246360-172.17.0.2-1733934224410:blk_1073741861_1044 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35157,DS-d85b2b77-d19f-4302-be38-ce7ae4e72a21,DISK], DatanodeInfoWithStorage[127.0.0.1:38801,DS-226c2206-3817-4f6d-b23c-9dac8d5d6b69,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35157,DS-d85b2b77-d19f-4302-be38-ce7ae4e72a21,DISK]) is bad.
2024-12-11T16:24:08,884 WARN  [Thread-946 {}] hdfs.DataStreamer(1850): Abandoning BP-1552246360-172.17.0.2-1733934224410:blk_1073741861_1044
2024-12-11T16:24:08,885 WARN  [Thread-946 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35157,DS-d85b2b77-d19f-4302-be38-ce7ae4e72a21,DISK]
2024-12-11T16:24:08,887 WARN  [Thread-946 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741862_1045
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:08,887 WARN  [Thread-946 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1552246360-172.17.0.2-1733934224410:blk_1073741862_1045 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38801,DS-226c2206-3817-4f6d-b23c-9dac8d5d6b69,DISK], DatanodeInfoWithStorage[127.0.0.1:40471,DS-c9a8be59-e598-41e5-9034-32246b7a6ac3,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38801,DS-226c2206-3817-4f6d-b23c-9dac8d5d6b69,DISK]) is bad.
2024-12-11T16:24:08,887 WARN  [Thread-946 {}] hdfs.DataStreamer(1850): Abandoning BP-1552246360-172.17.0.2-1733934224410:blk_1073741862_1045
2024-12-11T16:24:08,888 WARN  [Thread-946 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38801,DS-226c2206-3817-4f6d-b23c-9dac8d5d6b69,DISK]
2024-12-11T16:24:08,889 WARN  [Thread-946 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741863_1046
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:08,889 WARN  [Thread-946 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1552246360-172.17.0.2-1733934224410:blk_1073741863_1046 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK], DatanodeInfoWithStorage[127.0.0.1:39649,DS-8e83c89e-91af-4855-9e85-7f0bb0de8fa8,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK]) is bad.
2024-12-11T16:24:08,889 WARN  [Thread-946 {}] hdfs.DataStreamer(1850): Abandoning BP-1552246360-172.17.0.2-1733934224410:blk_1073741863_1046
2024-12-11T16:24:08,890 WARN  [Thread-946 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK]
2024-12-11T16:24:08,892 WARN  [Thread-946 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741864_1047
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:39649
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:08,892 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-501712066_22 at /127.0.0.1:33872 [Receiving block BP-1552246360-172.17.0.2-1733934224410:blk_1073741864_1047] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/cluster_7052d23d-6944-2d6a-b15d-561f403fc1d1/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/cluster_7052d23d-6944-2d6a-b15d-561f403fc1d1/data/data10]'}, localName='127.0.0.1:40471', datanodeUuid='852f5a8b-0ced-4a59-b572-6ce7ea337674', xmitsInProgress=0}:Exception transferring block BP-1552246360-172.17.0.2-1733934224410:blk_1073741864_1047 to mirror 127.0.0.1:39649
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T16:24:08,892 WARN  [Thread-946 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1552246360-172.17.0.2-1733934224410:blk_1073741864_1047 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40471,DS-c9a8be59-e598-41e5-9034-32246b7a6ac3,DISK], DatanodeInfoWithStorage[127.0.0.1:39649,DS-8e83c89e-91af-4855-9e85-7f0bb0de8fa8,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:39649,DS-8e83c89e-91af-4855-9e85-7f0bb0de8fa8,DISK]) is bad.
2024-12-11T16:24:08,892 WARN  [Thread-946 {}] hdfs.DataStreamer(1850): Abandoning BP-1552246360-172.17.0.2-1733934224410:blk_1073741864_1047
2024-12-11T16:24:08,892 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-501712066_22 at /127.0.0.1:33872 [Receiving block BP-1552246360-172.17.0.2-1733934224410:blk_1073741864_1047] {}] datanode.BlockReceiver(316): Block 1073741864 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-11T16:24:08,892 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-501712066_22 at /127.0.0.1:33872 [Receiving block BP-1552246360-172.17.0.2-1733934224410:blk_1073741864_1047] {}] datanode.DataXceiver(331): 127.0.0.1:40471:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:33872 dst: /127.0.0.1:40471
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T16:24:08,893 WARN  [Thread-946 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39649,DS-8e83c89e-91af-4855-9e85-7f0bb0de8fa8,DISK]
2024-12-11T16:24:08,893 WARN  [IPC Server handler 3 on default port 34525 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-11T16:24:08,894 WARN  [IPC Server handler 3 on default port 34525 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-11T16:24:08,894 WARN  [IPC Server handler 3 on default port 34525 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-11T16:24:08,896 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40471 is added to blk_1073741865_1048 (size=17994)
2024-12-11T16:24:08,925 INFO  [regionserver/7446a65196a1:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:09,111 WARN  [regionserver/7446a65196a1:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40471,DS-c9a8be59-e598-41e5-9034-32246b7a6ac3,DISK]]
2024-12-11T16:24:09,112 INFO  [regionserver/7446a65196a1:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:09,112 DEBUG [regionserver/7446a65196a1:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 7446a65196a1%2C45947%2C1733934227248:(num 1733934247093) roll requested
2024-12-11T16:24:09,113 INFO  [regionserver/7446a65196a1:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7446a65196a1%2C45947%2C1733934227248.1733934249112
2024-12-11T16:24:09,119 WARN  [Thread-951 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741866_1049
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:09,119 WARN  [Thread-951 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1552246360-172.17.0.2-1733934224410:blk_1073741866_1049 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35157,DS-d85b2b77-d19f-4302-be38-ce7ae4e72a21,DISK], DatanodeInfoWithStorage[127.0.0.1:39649,DS-8e83c89e-91af-4855-9e85-7f0bb0de8fa8,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35157,DS-d85b2b77-d19f-4302-be38-ce7ae4e72a21,DISK]) is bad.
2024-12-11T16:24:09,119 WARN  [Thread-951 {}] hdfs.DataStreamer(1850): Abandoning BP-1552246360-172.17.0.2-1733934224410:blk_1073741866_1049
2024-12-11T16:24:09,120 WARN  [Thread-951 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35157,DS-d85b2b77-d19f-4302-be38-ce7ae4e72a21,DISK]
2024-12-11T16:24:09,122 WARN  [Thread-951 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741867_1050
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:09,122 WARN  [Thread-951 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1552246360-172.17.0.2-1733934224410:blk_1073741867_1050 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39649,DS-8e83c89e-91af-4855-9e85-7f0bb0de8fa8,DISK], DatanodeInfoWithStorage[127.0.0.1:38801,DS-226c2206-3817-4f6d-b23c-9dac8d5d6b69,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39649,DS-8e83c89e-91af-4855-9e85-7f0bb0de8fa8,DISK]) is bad.
2024-12-11T16:24:09,122 WARN  [Thread-951 {}] hdfs.DataStreamer(1850): Abandoning BP-1552246360-172.17.0.2-1733934224410:blk_1073741867_1050
2024-12-11T16:24:09,123 WARN  [Thread-951 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39649,DS-8e83c89e-91af-4855-9e85-7f0bb0de8fa8,DISK]
2024-12-11T16:24:09,125 WARN  [Thread-951 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741868_1051
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:09,125 WARN  [Thread-951 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1552246360-172.17.0.2-1733934224410:blk_1073741868_1051 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK], DatanodeInfoWithStorage[127.0.0.1:40471,DS-c9a8be59-e598-41e5-9034-32246b7a6ac3,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK]) is bad.
2024-12-11T16:24:09,125 WARN  [Thread-951 {}] hdfs.DataStreamer(1850): Abandoning BP-1552246360-172.17.0.2-1733934224410:blk_1073741868_1051
2024-12-11T16:24:09,126 WARN  [Thread-951 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK]
2024-12-11T16:24:09,129 WARN  [Thread-951 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741869_1052
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:38801
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:09,129 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-501712066_22 at /127.0.0.1:33882 [Receiving block BP-1552246360-172.17.0.2-1733934224410:blk_1073741869_1052] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/cluster_7052d23d-6944-2d6a-b15d-561f403fc1d1/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/cluster_7052d23d-6944-2d6a-b15d-561f403fc1d1/data/data10]'}, localName='127.0.0.1:40471', datanodeUuid='852f5a8b-0ced-4a59-b572-6ce7ea337674', xmitsInProgress=0}:Exception transferring block BP-1552246360-172.17.0.2-1733934224410:blk_1073741869_1052 to mirror 127.0.0.1:38801
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T16:24:09,129 WARN  [Thread-951 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1552246360-172.17.0.2-1733934224410:blk_1073741869_1052 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40471,DS-c9a8be59-e598-41e5-9034-32246b7a6ac3,DISK], DatanodeInfoWithStorage[127.0.0.1:38801,DS-226c2206-3817-4f6d-b23c-9dac8d5d6b69,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:38801,DS-226c2206-3817-4f6d-b23c-9dac8d5d6b69,DISK]) is bad.
2024-12-11T16:24:09,129 WARN  [Thread-951 {}] hdfs.DataStreamer(1850): Abandoning BP-1552246360-172.17.0.2-1733934224410:blk_1073741869_1052
2024-12-11T16:24:09,129 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-501712066_22 at /127.0.0.1:33882 [Receiving block BP-1552246360-172.17.0.2-1733934224410:blk_1073741869_1052] {}] datanode.BlockReceiver(316): Block 1073741869 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-12-11T16:24:09,129 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-501712066_22 at /127.0.0.1:33882 [Receiving block BP-1552246360-172.17.0.2-1733934224410:blk_1073741869_1052] {}] datanode.DataXceiver(331): 127.0.0.1:40471:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:33882 dst: /127.0.0.1:40471
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T16:24:09,130 WARN  [Thread-951 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38801,DS-226c2206-3817-4f6d-b23c-9dac8d5d6b69,DISK]
2024-12-11T16:24:09,131 WARN  [IPC Server handler 0 on default port 34525 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-11T16:24:09,132 WARN  [IPC Server handler 0 on default port 34525 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-11T16:24:09,132 WARN  [IPC Server handler 0 on default port 34525 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-11T16:24:09,136 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:24:09,136 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:24:09,136 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:24:09,136 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:24:09,136 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:24:09,136 INFO  [regionserver/7446a65196a1:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.1733934247093 with entries=13, filesize=12.50 KB; new WAL /user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.1733934249112
2024-12-11T16:24:09,137 DEBUG [regionserver/7446a65196a1:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39651:39651)]
2024-12-11T16:24:09,137 DEBUG [regionserver/7446a65196a1:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.1733934227980 is not closed yet, will try archiving it next time
2024-12-11T16:24:09,137 DEBUG [regionserver/7446a65196a1:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.1733934247093 is not closed yet, will try archiving it next time
2024-12-11T16:24:09,138 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40471 is added to blk_1073741855_1038 (size=12810)
2024-12-11T16:24:09,305 DEBUG [RS:0;7446a65196a1:45947-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/.tmp/info/fca5937ef3da4f0cb79cd7baff0186e5 as hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/info/fca5937ef3da4f0cb79cd7baff0186e5
2024-12-11T16:24:09,314 INFO  [RS:0;7446a65196a1:45947-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96a752a5036dfbc63de411b3f9c85ba7/info of 96a752a5036dfbc63de411b3f9c85ba7 into fca5937ef3da4f0cb79cd7baff0186e5(size=17.6 K), total size for store is 17.6 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-11T16:24:09,314 DEBUG [RS:0;7446a65196a1:45947-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96a752a5036dfbc63de411b3f9c85ba7:

2024-12-11T16:24:09,315 INFO  [RS:0;7446a65196a1:45947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1733934228960.96a752a5036dfbc63de411b3f9c85ba7., storeName=96a752a5036dfbc63de411b3f9c85ba7/info, priority=13, startTime=1733934248865; duration=0sec
2024-12-11T16:24:09,315 DEBUG [RS:0;7446a65196a1:45947-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K
2024-12-11T16:24:09,315 DEBUG [RS:0;7446a65196a1:45947-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-11T16:24:09,315 DEBUG [RS:0;7446a65196a1:45947-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/info/fca5937ef3da4f0cb79cd7baff0186e5 because midkey is the same as first or last row
2024-12-11T16:24:09,315 DEBUG [RS:0;7446a65196a1:45947-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K
2024-12-11T16:24:09,315 DEBUG [RS:0;7446a65196a1:45947-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-11T16:24:09,315 DEBUG [RS:0;7446a65196a1:45947-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/info/fca5937ef3da4f0cb79cd7baff0186e5 because midkey is the same as first or last row
2024-12-11T16:24:09,315 DEBUG [RS:0;7446a65196a1:45947-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K
2024-12-11T16:24:09,315 DEBUG [RS:0;7446a65196a1:45947-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-11T16:24:09,315 DEBUG [RS:0;7446a65196a1:45947-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/info/fca5937ef3da4f0cb79cd7baff0186e5 because midkey is the same as first or last row
2024-12-11T16:24:09,315 DEBUG [RS:0;7446a65196a1:45947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-11T16:24:09,315 DEBUG [RS:0;7446a65196a1:45947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96a752a5036dfbc63de411b3f9c85ba7:info
2024-12-11T16:24:09,425 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:09,539 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.1733934227980 is not closed yet, will try archiving it next time
2024-12-11T16:24:09,672 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@703dee58[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:40471, datanodeUuid=852f5a8b-0ced-4a59-b572-6ce7ea337674, infoPort=39651, infoSecurePort=0, ipcPort=42181, storageInfo=lv=-57;cid=testClusterID;nsid=1545217681;c=1733934224410):Failed to transfer BP-1552246360-172.17.0.2-1733934224410:blk_1073741845_1028 to 127.0.0.1:44567 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T16:24:09,672 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@7efa1e2f[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:40471, datanodeUuid=852f5a8b-0ced-4a59-b572-6ce7ea337674, infoPort=39651, infoSecurePort=0, ipcPort=42181, storageInfo=lv=-57;cid=testClusterID;nsid=1545217681;c=1733934224410):Failed to transfer BP-1552246360-172.17.0.2-1733934224410:blk_1073741850_1033 to 127.0.0.1:38801 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T16:24:09,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45947 {}] regionserver.HRegion(8855): Flush requested on 96a752a5036dfbc63de411b3f9c85ba7
2024-12-11T16:24:09,844 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96a752a5036dfbc63de411b3f9c85ba7 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-12-11T16:24:09,850 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/.tmp/info/4ed0542ab69c4b2d9f40f8ce2e29ec00 is 1079, key is tmprow/info:/1733934249842/Put/seqid=0
2024-12-11T16:24:09,853 WARN  [Thread-960 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741871_1054
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:09,854 WARN  [Thread-960 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1552246360-172.17.0.2-1733934224410:blk_1073741871_1054 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK], DatanodeInfoWithStorage[127.0.0.1:39649,DS-8e83c89e-91af-4855-9e85-7f0bb0de8fa8,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK]) is bad.
2024-12-11T16:24:09,854 WARN  [Thread-960 {}] hdfs.DataStreamer(1850): Abandoning BP-1552246360-172.17.0.2-1733934224410:blk_1073741871_1054
2024-12-11T16:24:09,855 WARN  [Thread-960 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK]
2024-12-11T16:24:09,856 WARN  [Thread-960 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741872_1055
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:09,856 WARN  [Thread-960 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1552246360-172.17.0.2-1733934224410:blk_1073741872_1055 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39649,DS-8e83c89e-91af-4855-9e85-7f0bb0de8fa8,DISK], DatanodeInfoWithStorage[127.0.0.1:40471,DS-c9a8be59-e598-41e5-9034-32246b7a6ac3,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39649,DS-8e83c89e-91af-4855-9e85-7f0bb0de8fa8,DISK]) is bad.
2024-12-11T16:24:09,856 WARN  [Thread-960 {}] hdfs.DataStreamer(1850): Abandoning BP-1552246360-172.17.0.2-1733934224410:blk_1073741872_1055
2024-12-11T16:24:09,857 WARN  [Thread-960 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39649,DS-8e83c89e-91af-4855-9e85-7f0bb0de8fa8,DISK]
2024-12-11T16:24:09,858 WARN  [Thread-960 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741873_1056
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:09,859 WARN  [Thread-960 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1552246360-172.17.0.2-1733934224410:blk_1073741873_1056 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38801,DS-226c2206-3817-4f6d-b23c-9dac8d5d6b69,DISK], DatanodeInfoWithStorage[127.0.0.1:40471,DS-c9a8be59-e598-41e5-9034-32246b7a6ac3,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38801,DS-226c2206-3817-4f6d-b23c-9dac8d5d6b69,DISK]) is bad.
2024-12-11T16:24:09,859 WARN  [Thread-960 {}] hdfs.DataStreamer(1850): Abandoning BP-1552246360-172.17.0.2-1733934224410:blk_1073741873_1056
2024-12-11T16:24:09,859 WARN  [Thread-960 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38801,DS-226c2206-3817-4f6d-b23c-9dac8d5d6b69,DISK]
2024-12-11T16:24:09,862 WARN  [Thread-960 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741874_1057
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:35157
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:09,862 WARN  [Thread-960 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1552246360-172.17.0.2-1733934224410:blk_1073741874_1057 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40471,DS-c9a8be59-e598-41e5-9034-32246b7a6ac3,DISK], DatanodeInfoWithStorage[127.0.0.1:35157,DS-d85b2b77-d19f-4302-be38-ce7ae4e72a21,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:35157,DS-d85b2b77-d19f-4302-be38-ce7ae4e72a21,DISK]) is bad.
2024-12-11T16:24:09,862 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-501712066_22 at /127.0.0.1:33910 [Receiving block BP-1552246360-172.17.0.2-1733934224410:blk_1073741874_1057] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/cluster_7052d23d-6944-2d6a-b15d-561f403fc1d1/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/cluster_7052d23d-6944-2d6a-b15d-561f403fc1d1/data/data10]'}, localName='127.0.0.1:40471', datanodeUuid='852f5a8b-0ced-4a59-b572-6ce7ea337674', xmitsInProgress=0}:Exception transferring block BP-1552246360-172.17.0.2-1733934224410:blk_1073741874_1057 to mirror 127.0.0.1:35157
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T16:24:09,862 WARN  [Thread-960 {}] hdfs.DataStreamer(1850): Abandoning BP-1552246360-172.17.0.2-1733934224410:blk_1073741874_1057
2024-12-11T16:24:09,863 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-501712066_22 at /127.0.0.1:33910 [Receiving block BP-1552246360-172.17.0.2-1733934224410:blk_1073741874_1057] {}] datanode.BlockReceiver(316): Block 1073741874 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-11T16:24:09,863 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-501712066_22 at /127.0.0.1:33910 [Receiving block BP-1552246360-172.17.0.2-1733934224410:blk_1073741874_1057] {}] datanode.DataXceiver(331): 127.0.0.1:40471:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:33910 dst: /127.0.0.1:40471
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T16:24:09,863 WARN  [Thread-960 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35157,DS-d85b2b77-d19f-4302-be38-ce7ae4e72a21,DISK]
2024-12-11T16:24:09,864 WARN  [IPC Server handler 4 on default port 34525 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-11T16:24:09,864 WARN  [IPC Server handler 4 on default port 34525 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-11T16:24:09,864 WARN  [IPC Server handler 4 on default port 34525 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-11T16:24:09,868 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40471 is added to blk_1073741875_1058 (size=6027)
2024-12-11T16:24:10,269 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=43 (bloomFilter=true), to=hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/.tmp/info/4ed0542ab69c4b2d9f40f8ce2e29ec00
2024-12-11T16:24:10,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/.tmp/info/4ed0542ab69c4b2d9f40f8ce2e29ec00 as hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/info/4ed0542ab69c4b2d9f40f8ce2e29ec00
2024-12-11T16:24:10,280 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/info/4ed0542ab69c4b2d9f40f8ce2e29ec00, entries=1, sequenceid=43, filesize=5.9 K
2024-12-11T16:24:10,281 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for 96a752a5036dfbc63de411b3f9c85ba7 in 438ms, sequenceid=43, compaction requested=false
2024-12-11T16:24:10,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96a752a5036dfbc63de411b3f9c85ba7:

2024-12-11T16:24:10,282 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=23.5 K, sizeToCheck=16.0 K
2024-12-11T16:24:10,282 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-11T16:24:10,282 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/info/fca5937ef3da4f0cb79cd7baff0186e5 because midkey is the same as first or last row
2024-12-11T16:24:10,671 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@703dee58[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:40471, datanodeUuid=852f5a8b-0ced-4a59-b572-6ce7ea337674, infoPort=39651, infoSecurePort=0, ipcPort=42181, storageInfo=lv=-57;cid=testClusterID;nsid=1545217681;c=1733934224410):Failed to transfer BP-1552246360-172.17.0.2-1733934224410:blk_1073741860_1043 to 127.0.0.1:35157 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T16:24:10,671 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@7efa1e2f[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:40471, datanodeUuid=852f5a8b-0ced-4a59-b572-6ce7ea337674, infoPort=39651, infoSecurePort=0, ipcPort=42181, storageInfo=lv=-57;cid=testClusterID;nsid=1545217681;c=1733934224410):Failed to transfer BP-1552246360-172.17.0.2-1733934224410:blk_1073741840_1023 to 127.0.0.1:38801 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T16:24:10,925 INFO  [regionserver/7446a65196a1:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:11,137 WARN  [regionserver/7446a65196a1:0.logRoller {}] wal.FSHLog(539): Too many consecutive RollWriter requests, it's a sign of the total number of live datanodes is lower than the tolerable replicas.
2024-12-11T16:24:11,137 INFO  [regionserver/7446a65196a1:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:11,269 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-11T16:24:11,273 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-11T16:24:11,274 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-11T16:24:11,275 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-11T16:24:11,275 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-11T16:24:11,283 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3955e9ae{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/hadoop.log.dir/,AVAILABLE}
2024-12-11T16:24:11,284 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@758da55c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-11T16:24:11,417 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7cac94da{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/java.io.tmpdir/jetty-localhost-40609-hadoop-hdfs-3_4_1-tests_jar-_-any-2263434390983273130/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T16:24:11,418 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@894b118{HTTP/1.1, (http/1.1)}{localhost:40609}
2024-12-11T16:24:11,418 INFO  [Time-limited test {}] server.Server(415): Started @142088ms
2024-12-11T16:24:11,419 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-11T16:24:11,425 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:11,790 WARN  [Thread-979 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-11T16:24:11,797 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf5ba5fe35825d7e9 with lease ID 0x625a5ee7777b3bf2: from storage DS-226c2206-3817-4f6d-b23c-9dac8d5d6b69 node DatanodeRegistration(127.0.0.1:43417, datanodeUuid=3250d228-4a4c-40f3-8d16-2b51b2a499ea, infoPort=38369, infoSecurePort=0, ipcPort=45233, storageInfo=lv=-57;cid=testClusterID;nsid=1545217681;c=1733934224410), blocks: 6, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-12-11T16:24:11,797 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf5ba5fe35825d7e9 with lease ID 0x625a5ee7777b3bf2: from storage DS-0fedcaca-d67f-4052-96cd-792e7942bfff node DatanodeRegistration(127.0.0.1:43417, datanodeUuid=3250d228-4a4c-40f3-8d16-2b51b2a499ea, infoPort=38369, infoSecurePort=0, ipcPort=45233, storageInfo=lv=-57;cid=testClusterID;nsid=1545217681;c=1733934224410), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-11T16:24:12,672 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@703dee58[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:40471, datanodeUuid=852f5a8b-0ced-4a59-b572-6ce7ea337674, infoPort=39651, infoSecurePort=0, ipcPort=42181, storageInfo=lv=-57;cid=testClusterID;nsid=1545217681;c=1733934224410):Failed to transfer BP-1552246360-172.17.0.2-1733934224410:blk_1073741865_1048 to 127.0.0.1:35157 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T16:24:12,677 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741855_1038 (size=12810)
2024-12-11T16:24:12,926 INFO  [regionserver/7446a65196a1:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:13,137 INFO  [regionserver/7446a65196a1:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:13,426 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:13,675 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741875_1058 (size=6027)
2024-12-11T16:24:14,926 INFO  [regionserver/7446a65196a1:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:15,138 INFO  [regionserver/7446a65196a1:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:15,426 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:16,554 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-11T16:24:16,926 INFO  [regionserver/7446a65196a1:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:17,138 INFO  [regionserver/7446a65196a1:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:17,427 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:17,657 ERROR [FSHLog-0-hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/MasterData-prefix:7446a65196a1,36693,1733934227006 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:17,657 WARN  [FSHLog-0-hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/MasterData-prefix:7446a65196a1,36693,1733934227006 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:17,658 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(198): WAL FSHLog 7446a65196a1%2C36693%2C1733934227006:(num 1733934227442) roll requested
2024-12-11T16:24:17,658 INFO  [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7446a65196a1%2C36693%2C1733934227006.1733934257658
2024-12-11T16:24:17,663 WARN  [Thread-1001 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741876_1059
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:35157
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:17,662 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-752322117_22 at /127.0.0.1:48584 [Receiving block BP-1552246360-172.17.0.2-1733934224410:blk_1073741876_1059] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/cluster_7052d23d-6944-2d6a-b15d-561f403fc1d1/data/data3, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/cluster_7052d23d-6944-2d6a-b15d-561f403fc1d1/data/data4]'}, localName='127.0.0.1:43417', datanodeUuid='3250d228-4a4c-40f3-8d16-2b51b2a499ea', xmitsInProgress=0}:Exception transferring block BP-1552246360-172.17.0.2-1733934224410:blk_1073741876_1059 to mirror 127.0.0.1:35157
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T16:24:17,663 WARN  [Thread-1001 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1552246360-172.17.0.2-1733934224410:blk_1073741876_1059 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43417,DS-226c2206-3817-4f6d-b23c-9dac8d5d6b69,DISK], DatanodeInfoWithStorage[127.0.0.1:35157,DS-d85b2b77-d19f-4302-be38-ce7ae4e72a21,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:35157,DS-d85b2b77-d19f-4302-be38-ce7ae4e72a21,DISK]) is bad.
2024-12-11T16:24:17,663 WARN  [Thread-1001 {}] hdfs.DataStreamer(1850): Abandoning BP-1552246360-172.17.0.2-1733934224410:blk_1073741876_1059
2024-12-11T16:24:17,663 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-752322117_22 at /127.0.0.1:48584 [Receiving block BP-1552246360-172.17.0.2-1733934224410:blk_1073741876_1059] {}] datanode.BlockReceiver(316): Block 1073741876 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-12-11T16:24:17,663 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-752322117_22 at /127.0.0.1:48584 [Receiving block BP-1552246360-172.17.0.2-1733934224410:blk_1073741876_1059] {}] datanode.DataXceiver(331): 127.0.0.1:43417:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:48584 dst: /127.0.0.1:43417
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T16:24:17,663 WARN  [Thread-1001 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35157,DS-d85b2b77-d19f-4302-be38-ce7ae4e72a21,DISK]
2024-12-11T16:24:17,665 WARN  [Thread-1001 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741877_1060
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:17,665 WARN  [Thread-1001 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1552246360-172.17.0.2-1733934224410:blk_1073741877_1060 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK], DatanodeInfoWithStorage[127.0.0.1:43417,DS-226c2206-3817-4f6d-b23c-9dac8d5d6b69,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK]) is bad.
2024-12-11T16:24:17,665 WARN  [Thread-1001 {}] hdfs.DataStreamer(1850): Abandoning BP-1552246360-172.17.0.2-1733934224410:blk_1073741877_1060
2024-12-11T16:24:17,666 WARN  [Thread-1001 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK]
2024-12-11T16:24:17,671 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:24:17,671 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:24:17,671 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:24:17,671 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:24:17,671 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:24:17,671 INFO  [master:store-WAL-Roller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/MasterData/WALs/7446a65196a1,36693,1733934227006/7446a65196a1%2C36693%2C1733934227006.1733934227442 with entries=54, filesize=26.67 KB; new WAL /user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/MasterData/WALs/7446a65196a1,36693,1733934227006/7446a65196a1%2C36693%2C1733934227006.1733934257658
2024-12-11T16:24:17,672 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:17,672 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:17,672 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/MasterData/WALs/7446a65196a1,36693,1733934227006/7446a65196a1%2C36693%2C1733934227006.1733934227442
2024-12-11T16:24:17,672 WARN  [IPC Server handler 1 on default port 34525 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/MasterData/WALs/7446a65196a1,36693,1733934227006/7446a65196a1%2C36693%2C1733934227006.1733934227442 has not been closed. Lease recovery is in progress. RecoveryId = 1062 for block blk_1073741830_1006
2024-12-11T16:24:17,673 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/MasterData/WALs/7446a65196a1,36693,1733934227006/7446a65196a1%2C36693%2C1733934227006.1733934227442 after 1ms
2024-12-11T16:24:17,675 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38369:38369),(127.0.0.1/127.0.0.1:39651:39651)]
2024-12-11T16:24:17,675 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(879): hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/MasterData/WALs/7446a65196a1,36693,1733934227006/7446a65196a1%2C36693%2C1733934227006.1733934227442 is not closed yet, will try archiving it next time
2024-12-11T16:24:18,927 INFO  [regionserver/7446a65196a1:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:19,138 INFO  [regionserver/7446a65196a1:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:20,927 INFO  [regionserver/7446a65196a1:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:21,139 INFO  [regionserver/7446a65196a1:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:21,674 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/MasterData/WALs/7446a65196a1,36693,1733934227006/7446a65196a1%2C36693%2C1733934227006.1733934227442 after 4002ms
2024-12-11T16:24:21,813 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@3e04089a {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1552246360-172.17.0.2-1733934224410:blk_1073741833_1009, datanode=DatanodeInfoWithStorage[127.0.0.1:44567,null,null])
java.net.ConnectException: Call From 7446a65196a1/172.17.0.2 to localhost:38073 failed on connection exception: java.net.ConnectException: Connection refused; For more details see:  http://wiki.apache.org/hadoop/ConnectionRefused
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:876) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:668) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
2024-12-11T16:24:21,819 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741833_1019 (size=455)
2024-12-11T16:24:22,751 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.1733934227980 to hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/oldWALs/7446a65196a1%2C45947%2C1733934227248.1733934227980
2024-12-11T16:24:22,752 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.1733934247093 to hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/oldWALs/7446a65196a1%2C45947%2C1733934227248.1733934247093
2024-12-11T16:24:22,928 INFO  [regionserver/7446a65196a1:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:23,139 INFO  [regionserver/7446a65196a1:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:24,793 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@40757799[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:43417, datanodeUuid=3250d228-4a4c-40f3-8d16-2b51b2a499ea, infoPort=38369, infoSecurePort=0, ipcPort=45233, storageInfo=lv=-57;cid=testClusterID;nsid=1545217681;c=1733934224410):Failed to transfer BP-1552246360-172.17.0.2-1733934224410:blk_1073741833_1019 to 127.0.0.1:39649 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T16:24:24,928 INFO  [regionserver/7446a65196a1:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:25,140 INFO  [regionserver/7446a65196a1:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:26,599 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7446a65196a1%2C45947%2C1733934227248.1733934266599
2024-12-11T16:24:26,602 WARN  [Thread-1013 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741879_1063
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:26,602 WARN  [Thread-1013 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1552246360-172.17.0.2-1733934224410:blk_1073741879_1063 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35157,DS-d85b2b77-d19f-4302-be38-ce7ae4e72a21,DISK], DatanodeInfoWithStorage[127.0.0.1:43417,DS-226c2206-3817-4f6d-b23c-9dac8d5d6b69,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35157,DS-d85b2b77-d19f-4302-be38-ce7ae4e72a21,DISK]) is bad.
2024-12-11T16:24:26,602 WARN  [Thread-1013 {}] hdfs.DataStreamer(1850): Abandoning BP-1552246360-172.17.0.2-1733934224410:blk_1073741879_1063
2024-12-11T16:24:26,603 WARN  [Thread-1013 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35157,DS-d85b2b77-d19f-4302-be38-ce7ae4e72a21,DISK]
2024-12-11T16:24:26,624 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:24:26,624 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:24:26,625 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:24:26,625 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:24:26,625 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:24:26,625 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.1733934249112 with entries=13, filesize=12.60 KB; new WAL /user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.1733934266599
2024-12-11T16:24:26,632 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40471 is added to blk_1073741870_1053 (size=12911)
2024-12-11T16:24:26,634 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39651:39651),(127.0.0.1/127.0.0.1:38369:38369)]
2024-12-11T16:24:26,634 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.1733934249112 is not closed yet, will try archiving it next time
2024-12-11T16:24:26,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45947 {}] regionserver.HRegion(8855): Flush requested on 96a752a5036dfbc63de411b3f9c85ba7
2024-12-11T16:24:26,638 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96a752a5036dfbc63de411b3f9c85ba7 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-12-11T16:24:26,657 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/.tmp/info/359214a4544a4514ab5a84571f1852f1 is 1080, key is row0013/info:/1733934266635/Put/seqid=0
2024-12-11T16:24:26,669 WARN  [Thread-1019 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741881_1065
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:26,670 WARN  [Thread-1019 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1552246360-172.17.0.2-1733934224410:blk_1073741881_1065 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35157,DS-d85b2b77-d19f-4302-be38-ce7ae4e72a21,DISK], DatanodeInfoWithStorage[127.0.0.1:43417,DS-226c2206-3817-4f6d-b23c-9dac8d5d6b69,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35157,DS-d85b2b77-d19f-4302-be38-ce7ae4e72a21,DISK]) is bad.
2024-12-11T16:24:26,670 WARN  [Thread-1019 {}] hdfs.DataStreamer(1850): Abandoning BP-1552246360-172.17.0.2-1733934224410:blk_1073741881_1065
2024-12-11T16:24:26,670 WARN  [Thread-1019 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35157,DS-d85b2b77-d19f-4302-be38-ce7ae4e72a21,DISK]
2024-12-11T16:24:26,682 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741882_1066 (size=7109)
2024-12-11T16:24:26,682 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40471 is added to blk_1073741882_1066 (size=7109)
2024-12-11T16:24:26,682 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/.tmp/info/359214a4544a4514ab5a84571f1852f1
2024-12-11T16:24:26,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/.tmp/info/359214a4544a4514ab5a84571f1852f1 as hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/info/359214a4544a4514ab5a84571f1852f1
2024-12-11T16:24:26,698 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/info/359214a4544a4514ab5a84571f1852f1, entries=2, sequenceid=53, filesize=6.9 K
2024-12-11T16:24:26,699 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7526, heapSize ~8.11 KB/8304, currentSize=10.51 KB/10759 for 96a752a5036dfbc63de411b3f9c85ba7 in 62ms, sequenceid=53, compaction requested=true
2024-12-11T16:24:26,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96a752a5036dfbc63de411b3f9c85ba7:

2024-12-11T16:24:26,700 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=30.4 K, sizeToCheck=16.0 K
2024-12-11T16:24:26,700 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-11T16:24:26,700 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/info/fca5937ef3da4f0cb79cd7baff0186e5 because midkey is the same as first or last row
2024-12-11T16:24:26,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96a752a5036dfbc63de411b3f9c85ba7:info, priority=-2147483648, current under compaction store size is 1
2024-12-11T16:24:26,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-11T16:24:26,700 DEBUG [RS:0;7446a65196a1:45947-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-11T16:24:26,702 DEBUG [RS:0;7446a65196a1:45947-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 31130 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-11T16:24:26,702 DEBUG [RS:0;7446a65196a1:45947-shortCompactions-0 {}] regionserver.HStore(1541): 96a752a5036dfbc63de411b3f9c85ba7/info is initiating minor compaction (all files)
2024-12-11T16:24:26,702 INFO  [RS:0;7446a65196a1:45947-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96a752a5036dfbc63de411b3f9c85ba7/info in TestLogRolling-testLogRollOnDatanodeDeath,,1733934228960.96a752a5036dfbc63de411b3f9c85ba7.
2024-12-11T16:24:26,702 INFO  [RS:0;7446a65196a1:45947-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/info/fca5937ef3da4f0cb79cd7baff0186e5, hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/info/4ed0542ab69c4b2d9f40f8ce2e29ec00, hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/info/359214a4544a4514ab5a84571f1852f1] into tmpdir=hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/.tmp, totalSize=30.4 K
2024-12-11T16:24:26,703 DEBUG [RS:0;7446a65196a1:45947-shortCompactions-0 {}] compactions.Compactor(225): Compacting fca5937ef3da4f0cb79cd7baff0186e5, keycount=12, bloomtype=ROW, size=17.6 K, encoding=NONE, compression=NONE, seqNum=32, earliestPutTs=1733934242744
2024-12-11T16:24:26,703 DEBUG [RS:0;7446a65196a1:45947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ed0542ab69c4b2d9f40f8ce2e29ec00, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1733934249842
2024-12-11T16:24:26,704 DEBUG [RS:0;7446a65196a1:45947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 359214a4544a4514ab5a84571f1852f1, keycount=2, bloomtype=ROW, size=6.9 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1733934251059
2024-12-11T16:24:26,724 INFO  [RS:0;7446a65196a1:45947-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96a752a5036dfbc63de411b3f9c85ba7#info#compaction#24 average throughput is 6.67 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-11T16:24:26,725 DEBUG [RS:0;7446a65196a1:45947-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/.tmp/info/f1b53b3576a342549230c1a877f2b4fe is 1080, key is row0002/info:/1733934242744/Put/seqid=0
2024-12-11T16:24:26,732 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40471 is added to blk_1073741883_1067 (size=19190)
2024-12-11T16:24:26,734 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741883_1067 (size=19190)
2024-12-11T16:24:26,743 DEBUG [RS:0;7446a65196a1:45947-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/.tmp/info/f1b53b3576a342549230c1a877f2b4fe as hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/info/f1b53b3576a342549230c1a877f2b4fe
2024-12-11T16:24:26,753 INFO  [RS:0;7446a65196a1:45947-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96a752a5036dfbc63de411b3f9c85ba7/info of 96a752a5036dfbc63de411b3f9c85ba7 into f1b53b3576a342549230c1a877f2b4fe(size=18.7 K), total size for store is 18.7 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-11T16:24:26,753 DEBUG [RS:0;7446a65196a1:45947-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96a752a5036dfbc63de411b3f9c85ba7:

2024-12-11T16:24:26,753 INFO  [RS:0;7446a65196a1:45947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1733934228960.96a752a5036dfbc63de411b3f9c85ba7., storeName=96a752a5036dfbc63de411b3f9c85ba7/info, priority=13, startTime=1733934266700; duration=0sec
2024-12-11T16:24:26,754 DEBUG [RS:0;7446a65196a1:45947-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=18.7 K, sizeToCheck=16.0 K
2024-12-11T16:24:26,754 DEBUG [RS:0;7446a65196a1:45947-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-11T16:24:26,754 DEBUG [RS:0;7446a65196a1:45947-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/info/f1b53b3576a342549230c1a877f2b4fe because midkey is the same as first or last row
2024-12-11T16:24:26,754 DEBUG [RS:0;7446a65196a1:45947-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=18.7 K, sizeToCheck=16.0 K
2024-12-11T16:24:26,754 DEBUG [RS:0;7446a65196a1:45947-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-11T16:24:26,754 DEBUG [RS:0;7446a65196a1:45947-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/info/f1b53b3576a342549230c1a877f2b4fe because midkey is the same as first or last row
2024-12-11T16:24:26,754 DEBUG [RS:0;7446a65196a1:45947-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=18.7 K, sizeToCheck=16.0 K
2024-12-11T16:24:26,754 DEBUG [RS:0;7446a65196a1:45947-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-11T16:24:26,754 DEBUG [RS:0;7446a65196a1:45947-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/info/f1b53b3576a342549230c1a877f2b4fe because midkey is the same as first or last row
2024-12-11T16:24:26,754 DEBUG [RS:0;7446a65196a1:45947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-11T16:24:26,754 DEBUG [RS:0;7446a65196a1:45947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96a752a5036dfbc63de411b3f9c85ba7:info
2024-12-11T16:24:26,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45947 {}] regionserver.HRegion(8855): Flush requested on 96a752a5036dfbc63de411b3f9c85ba7
2024-12-11T16:24:26,877 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96a752a5036dfbc63de411b3f9c85ba7 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB
2024-12-11T16:24:26,883 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/.tmp/info/64e715ef6d1948c9901610837a979209 is 1080, key is row0014/info:/1733934266640/Put/seqid=0
2024-12-11T16:24:26,890 WARN  [Thread-1035 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741884_1068
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:35157
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:26,889 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-501712066_22 at /127.0.0.1:58456 [Receiving block BP-1552246360-172.17.0.2-1733934224410:blk_1073741884_1068] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/cluster_7052d23d-6944-2d6a-b15d-561f403fc1d1/data/data3, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/cluster_7052d23d-6944-2d6a-b15d-561f403fc1d1/data/data4]'}, localName='127.0.0.1:43417', datanodeUuid='3250d228-4a4c-40f3-8d16-2b51b2a499ea', xmitsInProgress=0}:Exception transferring block BP-1552246360-172.17.0.2-1733934224410:blk_1073741884_1068 to mirror 127.0.0.1:35157
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T16:24:26,890 WARN  [Thread-1035 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1552246360-172.17.0.2-1733934224410:blk_1073741884_1068 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43417,DS-226c2206-3817-4f6d-b23c-9dac8d5d6b69,DISK], DatanodeInfoWithStorage[127.0.0.1:35157,DS-d85b2b77-d19f-4302-be38-ce7ae4e72a21,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:35157,DS-d85b2b77-d19f-4302-be38-ce7ae4e72a21,DISK]) is bad.
2024-12-11T16:24:26,890 WARN  [Thread-1035 {}] hdfs.DataStreamer(1850): Abandoning BP-1552246360-172.17.0.2-1733934224410:blk_1073741884_1068
2024-12-11T16:24:26,890 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-501712066_22 at /127.0.0.1:58456 [Receiving block BP-1552246360-172.17.0.2-1733934224410:blk_1073741884_1068] {}] datanode.BlockReceiver(316): Block 1073741884 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-11T16:24:26,890 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-501712066_22 at /127.0.0.1:58456 [Receiving block BP-1552246360-172.17.0.2-1733934224410:blk_1073741884_1068] {}] datanode.DataXceiver(331): 127.0.0.1:43417:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:58456 dst: /127.0.0.1:43417
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T16:24:26,891 WARN  [Thread-1035 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35157,DS-d85b2b77-d19f-4302-be38-ce7ae4e72a21,DISK]
2024-12-11T16:24:26,896 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40471 is added to blk_1073741885_1069 (size=15737)
2024-12-11T16:24:26,898 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/.tmp/info/64e715ef6d1948c9901610837a979209
2024-12-11T16:24:26,898 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741885_1069 (size=15737)
2024-12-11T16:24:26,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/.tmp/info/64e715ef6d1948c9901610837a979209 as hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/info/64e715ef6d1948c9901610837a979209
2024-12-11T16:24:26,913 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/info/64e715ef6d1948c9901610837a979209, entries=10, sequenceid=68, filesize=15.4 K
2024-12-11T16:24:26,914 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11834, heapSize ~12.61 KB/12912, currentSize=0 B/0 for 96a752a5036dfbc63de411b3f9c85ba7 in 37ms, sequenceid=68, compaction requested=false
2024-12-11T16:24:26,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96a752a5036dfbc63de411b3f9c85ba7:

2024-12-11T16:24:26,915 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=34.1 K, sizeToCheck=16.0 K
2024-12-11T16:24:26,915 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-11T16:24:26,915 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/info/f1b53b3576a342549230c1a877f2b4fe because midkey is the same as first or last row
2024-12-11T16:24:26,928 INFO  [regionserver/7446a65196a1:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:27,028 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.1733934249112 to hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/oldWALs/7446a65196a1%2C45947%2C1733934227248.1733934249112
2024-12-11T16:24:27,140 INFO  [regionserver/7446a65196a1:0.logRoller {}] wal.FSHLog(556): LowReplication-Roller was enabled.
2024-12-11T16:24:27,140 INFO  [regionserver/7446a65196a1:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:27,281 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-12-11T16:24:27,282 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-11T16:24:27,282 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-11T16:24:27,282 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T16:24:27,282 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T16:24:27,282 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-12-11T16:24:27,283 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-12-11T16:24:27,283 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1809187819, stopped=false
2024-12-11T16:24:27,283 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=7446a65196a1,36693,1733934227006
2024-12-11T16:24:27,294 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42595-0x10015be5b1a0002, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-11T16:24:27,294 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36693-0x10015be5b1a0000, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-11T16:24:27,294 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45947-0x10015be5b1a0001, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-11T16:24:27,294 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36693-0x10015be5b1a0000, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:24:27,294 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42595-0x10015be5b1a0002, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:24:27,294 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45947-0x10015be5b1a0001, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:24:27,294 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-11T16:24:27,295 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-11T16:24:27,295 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-11T16:24:27,295 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T16:24:27,296 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45947-0x10015be5b1a0001, quorum=127.0.0.1:53294, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-11T16:24:27,296 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '7446a65196a1,45947,1733934227248' *****
2024-12-11T16:24:27,296 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42595-0x10015be5b1a0002, quorum=127.0.0.1:53294, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-11T16:24:27,296 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-12-11T16:24:27,296 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36693-0x10015be5b1a0000, quorum=127.0.0.1:53294, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-11T16:24:27,296 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '7446a65196a1,42595,1733934228675' *****
2024-12-11T16:24:27,296 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-12-11T16:24:27,297 INFO  [RS:0;7446a65196a1:45947 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-11T16:24:27,297 INFO  [RS:1;7446a65196a1:42595 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-11T16:24:27,298 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-12-11T16:24:27,298 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-12-11T16:24:27,298 INFO  [RS:1;7446a65196a1:42595 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-11T16:24:27,298 INFO  [RS:1;7446a65196a1:42595 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-11T16:24:27,298 INFO  [RS:0;7446a65196a1:45947 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-11T16:24:27,298 INFO  [RS:1;7446a65196a1:42595 {}] regionserver.HRegionServer(959): stopping server 7446a65196a1,42595,1733934228675
2024-12-11T16:24:27,298 INFO  [RS:0;7446a65196a1:45947 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-11T16:24:27,298 INFO  [RS:1;7446a65196a1:42595 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-11T16:24:27,298 INFO  [RS:0;7446a65196a1:45947 {}] regionserver.HRegionServer(3091): Received CLOSE for 96a752a5036dfbc63de411b3f9c85ba7
2024-12-11T16:24:27,298 INFO  [RS:1;7446a65196a1:42595 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;7446a65196a1:42595.
2024-12-11T16:24:27,299 DEBUG [RS:1;7446a65196a1:42595 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-11T16:24:27,299 INFO  [RS:0;7446a65196a1:45947 {}] regionserver.HRegionServer(959): stopping server 7446a65196a1,45947,1733934227248
2024-12-11T16:24:27,299 INFO  [RS:0;7446a65196a1:45947 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-11T16:24:27,299 DEBUG [RS:1;7446a65196a1:42595 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T16:24:27,299 INFO  [RS:0;7446a65196a1:45947 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;7446a65196a1:45947.
2024-12-11T16:24:27,299 INFO  [RS:1;7446a65196a1:42595 {}] regionserver.HRegionServer(976): stopping server 7446a65196a1,42595,1733934228675; all regions closed.
2024-12-11T16:24:27,299 DEBUG [RS:0;7446a65196a1:45947 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-11T16:24:27,299 DEBUG [RS:0;7446a65196a1:45947 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T16:24:27,299 DEBUG [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 96a752a5036dfbc63de411b3f9c85ba7, disabling compactions & flushes
2024-12-11T16:24:27,299 INFO  [RS:0;7446a65196a1:45947 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-11T16:24:27,299 INFO  [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1733934228960.96a752a5036dfbc63de411b3f9c85ba7.
2024-12-11T16:24:27,299 INFO  [RS:0;7446a65196a1:45947 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-11T16:24:27,300 INFO  [RS:0;7446a65196a1:45947 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-11T16:24:27,300 DEBUG [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1733934228960.96a752a5036dfbc63de411b3f9c85ba7.
2024-12-11T16:24:27,300 INFO  [RS:0;7446a65196a1:45947 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-12-11T16:24:27,300 DEBUG [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1733934228960.96a752a5036dfbc63de411b3f9c85ba7. after waiting 0 ms
2024-12-11T16:24:27,300 DEBUG [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1733934228960.96a752a5036dfbc63de411b3f9c85ba7.
2024-12-11T16:24:27,300 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:24:27,300 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:24:27,300 INFO  [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 96a752a5036dfbc63de411b3f9c85ba7 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-12-11T16:24:27,300 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:24:27,300 INFO  [RS:0;7446a65196a1:45947 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-12-11T16:24:27,300 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:24:27,300 DEBUG [RS:0;7446a65196a1:45947 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 96a752a5036dfbc63de411b3f9c85ba7=TestLogRolling-testLogRollOnDatanodeDeath,,1733934228960.96a752a5036dfbc63de411b3f9c85ba7.}
2024-12-11T16:24:27,300 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:24:27,300 DEBUG [RS:0;7446a65196a1:45947 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 96a752a5036dfbc63de411b3f9c85ba7
2024-12-11T16:24:27,300 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-11T16:24:27,300 INFO  [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-11T16:24:27,300 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-11T16:24:27,300 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-11T16:24:27,300 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-11T16:24:27,301 INFO  [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.71 KB heapSize=3.75 KB
2024-12-11T16:24:27,301 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:27,301 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:27,301 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
2024-12-11T16:24:27,301 ERROR [FSHLog-0-hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee-prefix:7446a65196a1,45947,1733934227248.meta {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:27,301 WARN  [FSHLog-0-hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee-prefix:7446a65196a1,45947,1733934227248.meta {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:27,301 DEBUG [regionserver/7446a65196a1:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 7446a65196a1%2C45947%2C1733934227248.meta:.meta(num 1733934228399) roll requested
2024-12-11T16:24:27,301 WARN  [IPC Server handler 1 on default port 34525 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064 has not been closed. Lease recovery is in progress. RecoveryId = 1070 for block blk_1073741837_1013
2024-12-11T16:24:27,302 INFO  [regionserver/7446a65196a1:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7446a65196a1%2C45947%2C1733934227248.meta.1733934267301.meta
2024-12-11T16:24:27,302 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064 after 1ms
2024-12-11T16:24:27,304 WARN  [Thread-1042 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741886_1071
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:27,305 WARN  [Thread-1042 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1552246360-172.17.0.2-1733934224410:blk_1073741886_1071 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35157,DS-d85b2b77-d19f-4302-be38-ce7ae4e72a21,DISK], DatanodeInfoWithStorage[127.0.0.1:43417,DS-226c2206-3817-4f6d-b23c-9dac8d5d6b69,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35157,DS-d85b2b77-d19f-4302-be38-ce7ae4e72a21,DISK]) is bad.
2024-12-11T16:24:27,305 WARN  [Thread-1042 {}] hdfs.DataStreamer(1850): Abandoning BP-1552246360-172.17.0.2-1733934224410:blk_1073741886_1071
2024-12-11T16:24:27,305 WARN  [Thread-1042 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35157,DS-d85b2b77-d19f-4302-be38-ce7ae4e72a21,DISK]
2024-12-11T16:24:27,306 DEBUG [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/.tmp/info/6ab245136ad34f61ad67332915908ae5 is 1079, key is tmprow/info:/1733934267079/Put/seqid=0
2024-12-11T16:24:27,308 WARN  [Thread-1043 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741888_1073
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:35157
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:27,308 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-501712066_22 at /127.0.0.1:36860 [Receiving block BP-1552246360-172.17.0.2-1733934224410:blk_1073741888_1073] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/cluster_7052d23d-6944-2d6a-b15d-561f403fc1d1/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/cluster_7052d23d-6944-2d6a-b15d-561f403fc1d1/data/data10]'}, localName='127.0.0.1:40471', datanodeUuid='852f5a8b-0ced-4a59-b572-6ce7ea337674', xmitsInProgress=0}:Exception transferring block BP-1552246360-172.17.0.2-1733934224410:blk_1073741888_1073 to mirror 127.0.0.1:35157
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T16:24:27,308 WARN  [Thread-1043 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1552246360-172.17.0.2-1733934224410:blk_1073741888_1073 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40471,DS-c9a8be59-e598-41e5-9034-32246b7a6ac3,DISK], DatanodeInfoWithStorage[127.0.0.1:35157,DS-d85b2b77-d19f-4302-be38-ce7ae4e72a21,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:35157,DS-d85b2b77-d19f-4302-be38-ce7ae4e72a21,DISK]) is bad.
2024-12-11T16:24:27,308 WARN  [Thread-1043 {}] hdfs.DataStreamer(1850): Abandoning BP-1552246360-172.17.0.2-1733934224410:blk_1073741888_1073
2024-12-11T16:24:27,308 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-501712066_22 at /127.0.0.1:36860 [Receiving block BP-1552246360-172.17.0.2-1733934224410:blk_1073741888_1073] {}] datanode.BlockReceiver(316): Block 1073741888 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-11T16:24:27,308 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-501712066_22 at /127.0.0.1:36860 [Receiving block BP-1552246360-172.17.0.2-1733934224410:blk_1073741888_1073] {}] datanode.DataXceiver(331): 127.0.0.1:40471:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:36860 dst: /127.0.0.1:40471
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T16:24:27,308 WARN  [Thread-1043 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35157,DS-d85b2b77-d19f-4302-be38-ce7ae4e72a21,DISK]
2024-12-11T16:24:27,312 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:24:27,312 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:24:27,312 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:24:27,312 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:24:27,312 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:24:27,312 INFO  [regionserver/7446a65196a1:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta with entries=8, filesize=2.33 KB; new WAL /user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934267301.meta
2024-12-11T16:24:27,313 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:27,313 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44567,DS-ea0e888f-b34d-4dd3-bb95-5725242e322a,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:27,313 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
2024-12-11T16:24:27,313 DEBUG [regionserver/7446a65196a1:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38369:38369),(127.0.0.1/127.0.0.1:39651:39651)]
2024-12-11T16:24:27,313 DEBUG [regionserver/7446a65196a1:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta is not closed yet, will try archiving it next time
2024-12-11T16:24:27,313 WARN  [IPC Server handler 0 on default port 34525 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta has not been closed. Lease recovery is in progress. RecoveryId = 1075 for block blk_1073741834_1010
2024-12-11T16:24:27,314 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta after 1ms
2024-12-11T16:24:27,314 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741889_1074 (size=6027)
2024-12-11T16:24:27,314 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40471 is added to blk_1073741889_1074 (size=6027)
2024-12-11T16:24:27,314 INFO  [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=72 (bloomFilter=true), to=hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/.tmp/info/6ab245136ad34f61ad67332915908ae5
2024-12-11T16:24:27,320 DEBUG [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/.tmp/info/6ab245136ad34f61ad67332915908ae5 as hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/info/6ab245136ad34f61ad67332915908ae5
2024-12-11T16:24:27,326 INFO  [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/info/6ab245136ad34f61ad67332915908ae5, entries=1, sequenceid=72, filesize=5.9 K
2024-12-11T16:24:27,327 INFO  [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1075, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 96a752a5036dfbc63de411b3f9c85ba7 in 27ms, sequenceid=72, compaction requested=true
2024-12-11T16:24:27,328 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733934228960.96a752a5036dfbc63de411b3f9c85ba7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/info/3afb4e2b0a8043cea9a75cbf8e8db744, hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/info/9120724ff6334105b29217fc805dd912, hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/info/fca5937ef3da4f0cb79cd7baff0186e5, hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/info/dfbc5851194d473991cff9e3fe8e2427, hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/info/4ed0542ab69c4b2d9f40f8ce2e29ec00, hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/info/359214a4544a4514ab5a84571f1852f1] to archive
2024-12-11T16:24:27,329 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733934228960.96a752a5036dfbc63de411b3f9c85ba7.-1 {}] backup.HFileArchiver(363): Archiving compacted files.
2024-12-11T16:24:27,329 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/hbase/meta/1588230740/.tmp/info/9426e8bfc7544660996bd0ad5663bf61 is 203, key is TestLogRolling-testLogRollOnDatanodeDeath,,1733934228960.96a752a5036dfbc63de411b3f9c85ba7./info:regioninfo/1733934229334/Put/seqid=0
2024-12-11T16:24:27,331 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/info/3afb4e2b0a8043cea9a75cbf8e8db744 to hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/info/3afb4e2b0a8043cea9a75cbf8e8db744
2024-12-11T16:24:27,333 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/info/dfbc5851194d473991cff9e3fe8e2427 to hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/info/dfbc5851194d473991cff9e3fe8e2427
2024-12-11T16:24:27,333 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/info/fca5937ef3da4f0cb79cd7baff0186e5 to hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/info/fca5937ef3da4f0cb79cd7baff0186e5
2024-12-11T16:24:27,333 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/info/9120724ff6334105b29217fc805dd912 to hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/info/9120724ff6334105b29217fc805dd912
2024-12-11T16:24:27,334 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/info/4ed0542ab69c4b2d9f40f8ce2e29ec00 to hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/info/4ed0542ab69c4b2d9f40f8ce2e29ec00
2024-12-11T16:24:27,334 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/info/359214a4544a4514ab5a84571f1852f1 to hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/info/359214a4544a4514ab5a84571f1852f1
2024-12-11T16:24:27,335 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733934228960.96a752a5036dfbc63de411b3f9c85ba7.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=7446a65196a1:36693 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 16 more
2024-12-11T16:24:27,335 WARN  [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733934228960.96a752a5036dfbc63de411b3f9c85ba7.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [3afb4e2b0a8043cea9a75cbf8e8db744=10347, 9120724ff6334105b29217fc805dd912=12506, fca5937ef3da4f0cb79cd7baff0186e5=17994, dfbc5851194d473991cff9e3fe8e2427=6027, 4ed0542ab69c4b2d9f40f8ce2e29ec00=6027, 359214a4544a4514ab5a84571f1852f1=7109]
2024-12-11T16:24:27,338 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40471 is added to blk_1073741890_1076 (size=7089)
2024-12-11T16:24:27,338 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741890_1076 (size=7089)
2024-12-11T16:24:27,338 INFO  [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.50 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/hbase/meta/1588230740/.tmp/info/9426e8bfc7544660996bd0ad5663bf61
2024-12-11T16:24:27,342 DEBUG [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/default/TestLogRolling-testLogRollOnDatanodeDeath/96a752a5036dfbc63de411b3f9c85ba7/recovered.edits/75.seqid, newMaxSeqId=75, maxSeqId=1
2024-12-11T16:24:27,342 INFO  [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1733934228960.96a752a5036dfbc63de411b3f9c85ba7.
2024-12-11T16:24:27,343 DEBUG [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 96a752a5036dfbc63de411b3f9c85ba7:
Waiting for close lock at 1733934267299Running coprocessor pre-close hooks at 1733934267299Disabling compacts and flushes for region at 1733934267299Disabling writes for close at 1733934267300 (+1 ms)Obtaining lock to block concurrent updates at 1733934267300Preparing flush snapshotting stores in 96a752a5036dfbc63de411b3f9c85ba7 at 1733934267300Finished memstore snapshotting TestLogRolling-testLogRollOnDatanodeDeath,,1733934228960.96a752a5036dfbc63de411b3f9c85ba7., syncing WAL and waiting on mvcc, flushsize=dataSize=1075, getHeapSize=1392, getOffHeapSize=0, getCellsCount=1 at 1733934267300Flushing stores of TestLogRolling-testLogRollOnDatanodeDeath,,1733934228960.96a752a5036dfbc63de411b3f9c85ba7. at 1733934267301 (+1 ms)Flushing 96a752a5036dfbc63de411b3f9c85ba7/info: creating writer at 1733934267301Flushing 96a752a5036dfbc63de411b3f9c85ba7/info: appending metadata at 1733934267305 (+4 ms)Flushing 96a752a5036dfbc63de411b3f9c85ba7/info: closing flushed file at 1733934267305Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@36cef059: reopening flushed file at 1733934267319 (+14 ms)Finished flush of dataSize ~1.05 KB/1075, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 96a752a5036dfbc63de411b3f9c85ba7 in 27ms, sequenceid=72, compaction requested=true at 1733934267327 (+8 ms)Writing region close event to WAL at 1733934267337 (+10 ms)Running coprocessor post-close hooks at 1733934267342 (+5 ms)Closed at 1733934267342
2024-12-11T16:24:27,343 DEBUG [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1733934228960.96a752a5036dfbc63de411b3f9c85ba7.
2024-12-11T16:24:27,361 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/hbase/meta/1588230740/.tmp/ns/6e2a7f2f74f543dfb83073c02e3a8a79 is 43, key is default/ns:d/1733934228508/Put/seqid=0
2024-12-11T16:24:27,364 WARN  [Thread-1062 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741891_1077
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:35157
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:27,364 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-501712066_22 at /127.0.0.1:58486 [Receiving block BP-1552246360-172.17.0.2-1733934224410:blk_1073741891_1077] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/cluster_7052d23d-6944-2d6a-b15d-561f403fc1d1/data/data3, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/cluster_7052d23d-6944-2d6a-b15d-561f403fc1d1/data/data4]'}, localName='127.0.0.1:43417', datanodeUuid='3250d228-4a4c-40f3-8d16-2b51b2a499ea', xmitsInProgress=0}:Exception transferring block BP-1552246360-172.17.0.2-1733934224410:blk_1073741891_1077 to mirror 127.0.0.1:35157
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T16:24:27,364 WARN  [Thread-1062 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1552246360-172.17.0.2-1733934224410:blk_1073741891_1077 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43417,DS-226c2206-3817-4f6d-b23c-9dac8d5d6b69,DISK], DatanodeInfoWithStorage[127.0.0.1:35157,DS-d85b2b77-d19f-4302-be38-ce7ae4e72a21,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:35157,DS-d85b2b77-d19f-4302-be38-ce7ae4e72a21,DISK]) is bad.
2024-12-11T16:24:27,365 WARN  [Thread-1062 {}] hdfs.DataStreamer(1850): Abandoning BP-1552246360-172.17.0.2-1733934224410:blk_1073741891_1077
2024-12-11T16:24:27,365 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-501712066_22 at /127.0.0.1:58486 [Receiving block BP-1552246360-172.17.0.2-1733934224410:blk_1073741891_1077] {}] datanode.BlockReceiver(316): Block 1073741891 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-11T16:24:27,365 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-501712066_22 at /127.0.0.1:58486 [Receiving block BP-1552246360-172.17.0.2-1733934224410:blk_1073741891_1077] {}] datanode.DataXceiver(331): 127.0.0.1:43417:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:58486 dst: /127.0.0.1:43417
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T16:24:27,366 WARN  [Thread-1062 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35157,DS-d85b2b77-d19f-4302-be38-ce7ae4e72a21,DISK]
2024-12-11T16:24:27,379 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40471 is added to blk_1073741892_1078 (size=5153)
2024-12-11T16:24:27,379 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741892_1078 (size=5153)
2024-12-11T16:24:27,500 DEBUG [RS:0;7446a65196a1:45947 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-12-11T16:24:27,701 DEBUG [RS:0;7446a65196a1:45947 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-12-11T16:24:27,777 INFO  [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/hbase/meta/1588230740/.tmp/ns/6e2a7f2f74f543dfb83073c02e3a8a79
2024-12-11T16:24:27,798 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/hbase/meta/1588230740/.tmp/table/9b88ec4c4b8c46269d1ee25314e519b2 is 77, key is TestLogRolling-testLogRollOnDatanodeDeath/table:state/1733934229345/Put/seqid=0
2024-12-11T16:24:27,799 WARN  [Thread-1069 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741893_1079
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:27,800 WARN  [Thread-1069 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1552246360-172.17.0.2-1733934224410:blk_1073741893_1079 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35157,DS-d85b2b77-d19f-4302-be38-ce7ae4e72a21,DISK], DatanodeInfoWithStorage[127.0.0.1:43417,DS-226c2206-3817-4f6d-b23c-9dac8d5d6b69,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35157,DS-d85b2b77-d19f-4302-be38-ce7ae4e72a21,DISK]) is bad.
2024-12-11T16:24:27,800 WARN  [Thread-1069 {}] hdfs.DataStreamer(1850): Abandoning BP-1552246360-172.17.0.2-1733934224410:blk_1073741893_1079
2024-12-11T16:24:27,800 WARN  [Thread-1069 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35157,DS-d85b2b77-d19f-4302-be38-ce7ae4e72a21,DISK]
2024-12-11T16:24:27,805 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741894_1080 (size=5424)
2024-12-11T16:24:27,805 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40471 is added to blk_1073741894_1080 (size=5424)
2024-12-11T16:24:27,805 INFO  [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=146 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/hbase/meta/1588230740/.tmp/table/9b88ec4c4b8c46269d1ee25314e519b2
2024-12-11T16:24:27,812 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/hbase/meta/1588230740/.tmp/info/9426e8bfc7544660996bd0ad5663bf61 as hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/hbase/meta/1588230740/info/9426e8bfc7544660996bd0ad5663bf61
2024-12-11T16:24:27,818 INFO  [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/hbase/meta/1588230740/info/9426e8bfc7544660996bd0ad5663bf61, entries=10, sequenceid=11, filesize=6.9 K
2024-12-11T16:24:27,819 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/hbase/meta/1588230740/.tmp/ns/6e2a7f2f74f543dfb83073c02e3a8a79 as hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/hbase/meta/1588230740/ns/6e2a7f2f74f543dfb83073c02e3a8a79
2024-12-11T16:24:27,825 INFO  [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/hbase/meta/1588230740/ns/6e2a7f2f74f543dfb83073c02e3a8a79, entries=2, sequenceid=11, filesize=5.0 K
2024-12-11T16:24:27,826 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/hbase/meta/1588230740/.tmp/table/9b88ec4c4b8c46269d1ee25314e519b2 as hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/hbase/meta/1588230740/table/9b88ec4c4b8c46269d1ee25314e519b2
2024-12-11T16:24:27,832 INFO  [regionserver/7446a65196a1:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-12-11T16:24:27,832 INFO  [regionserver/7446a65196a1:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-12-11T16:24:27,833 INFO  [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/hbase/meta/1588230740/table/9b88ec4c4b8c46269d1ee25314e519b2, entries=2, sequenceid=11, filesize=5.3 K
2024-12-11T16:24:27,835 INFO  [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.71 KB/1752, heapSize ~3.45 KB/3536, currentSize=0 B/0 for 1588230740 in 534ms, sequenceid=11, compaction requested=false
2024-12-11T16:24:27,836 INFO  [regionserver/7446a65196a1:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-11T16:24:27,840 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-12-11T16:24:27,841 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-11T16:24:27,841 INFO  [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-11T16:24:27,842 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733934267300Running coprocessor pre-close hooks at 1733934267300Disabling compacts and flushes for region at 1733934267300Disabling writes for close at 1733934267300Obtaining lock to block concurrent updates at 1733934267301 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1733934267301Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1752, getHeapSize=3776, getOffHeapSize=0, getCellsCount=14 at 1733934267301Flushing stores of hbase:meta,,1.1588230740 at 1733934267314 (+13 ms)Flushing 1588230740/info: creating writer at 1733934267314Flushing 1588230740/info: appending metadata at 1733934267328 (+14 ms)Flushing 1588230740/info: closing flushed file at 1733934267329 (+1 ms)Flushing 1588230740/ns: creating writer at 1733934267348 (+19 ms)Flushing 1588230740/ns: appending metadata at 1733934267360 (+12 ms)Flushing 1588230740/ns: closing flushed file at 1733934267361 (+1 ms)Flushing 1588230740/table: creating writer at 1733934267785 (+424 ms)Flushing 1588230740/table: appending metadata at 1733934267797 (+12 ms)Flushing 1588230740/table: closing flushed file at 1733934267797Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3af3825e: reopening flushed file at 1733934267811 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@25ee8a95: reopening flushed file at 1733934267818 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@19341a67: reopening flushed file at 1733934267825 (+7 ms)Finished flush of dataSize ~1.71 KB/1752, heapSize ~3.45 KB/3536, currentSize=0 B/0 for 1588230740 in 534ms, sequenceid=11, compaction requested=false at 1733934267835 (+10 ms)Writing region close event to WAL at 1733934267836 (+1 ms)Running coprocessor post-close hooks at 1733934267841 (+5 ms)Closed at 1733934267841
2024-12-11T16:24:27,842 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-12-11T16:24:27,901 INFO  [RS:0;7446a65196a1:45947 {}] regionserver.HRegionServer(976): stopping server 7446a65196a1,45947,1733934227248; all regions closed.
2024-12-11T16:24:27,902 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:24:27,902 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:24:27,902 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:24:27,902 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:24:27,902 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:24:27,904 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40471 is added to blk_1073741887_1072 (size=825)
2024-12-11T16:24:27,904 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741887_1072 (size=825)
2024-12-11T16:24:27,925 INFO  [regionserver/7446a65196a1:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-12-11T16:24:27,925 INFO  [regionserver/7446a65196a1:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-12-11T16:24:28,765 INFO  [master/7446a65196a1:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-12-11T16:24:28,766 INFO  [master/7446a65196a1:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-12-11T16:24:28,793 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@3d02e272[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:43417, datanodeUuid=3250d228-4a4c-40f3-8d16-2b51b2a499ea, infoPort=38369, infoSecurePort=0, ipcPort=45233, storageInfo=lv=-57;cid=testClusterID;nsid=1545217681;c=1733934224410):Failed to transfer BP-1552246360-172.17.0.2-1733934224410:blk_1073741832_1008 to 127.0.0.1:35157 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T16:24:28,795 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40471 is added to blk_1073741836_1012 (size=76)
2024-12-11T16:24:28,927 INFO  [regionserver/7446a65196a1:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-11T16:24:29,797 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40471 is added to blk_1073741828_1004 (size=1189)
2024-12-11T16:24:29,798 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40471 is added to blk_1073741826_1002 (size=42)
2024-12-11T16:24:30,680 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741870_1053 (size=12911)
2024-12-11T16:24:30,795 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40471 is added to blk_1073741827_1003 (size=196)
2024-12-11T16:24:30,796 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40471 is added to blk_1073741829_1005 (size=34)
2024-12-11T16:24:31,303 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064 after 4002ms
2024-12-11T16:24:31,315 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta after 4002ms
2024-12-11T16:24:31,807 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40471 is added to blk_1073741825_1001 (size=7)
2024-12-11T16:24:31,818 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@38eb7426 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1552246360-172.17.0.2-1733934224410:blk_1073741830_1006, datanode=DatanodeInfoWithStorage[127.0.0.1:44567,null,null])
java.net.ConnectException: Call From 7446a65196a1/172.17.0.2 to localhost:38073 failed on connection exception: java.net.ConnectException: Connection refused; For more details see:  http://wiki.apache.org/hadoop/ConnectionRefused
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:876) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:668) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
2024-12-11T16:24:32,301 ERROR [WAL-Shutdown-0 {}] wal.AbstractFSWAL(2118): We have waited 5 seconds but the close of async writer doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.fshlog.wait.on.shutdown.seconds"
2024-12-11T16:24:32,323 DEBUG [RS:1;7446a65196a1:42595 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/oldWALs
2024-12-11T16:24:32,324 INFO  [RS:1;7446a65196a1:42595 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 7446a65196a1%2C42595%2C1733934228675:(num 1733934229064)
2024-12-11T16:24:32,324 DEBUG [RS:1;7446a65196a1:42595 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T16:24:32,324 INFO  [RS:1;7446a65196a1:42595 {}] regionserver.LeaseManager(133): Closed leases
2024-12-11T16:24:32,325 INFO  [RS:1;7446a65196a1:42595 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-11T16:24:32,325 INFO  [RS:1;7446a65196a1:42595 {}] hbase.ChoreService(370): Chore service for: regionserver/7446a65196a1:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-12-11T16:24:32,326 INFO  [RS:1;7446a65196a1:42595 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-11T16:24:32,326 INFO  [RS:1;7446a65196a1:42595 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-11T16:24:32,326 INFO  [regionserver/7446a65196a1:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-11T16:24:32,326 INFO  [RS:1;7446a65196a1:42595 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-11T16:24:32,326 INFO  [RS:1;7446a65196a1:42595 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-11T16:24:32,326 INFO  [RS:1;7446a65196a1:42595 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42595
2024-12-11T16:24:32,343 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:24:32,354 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:24:32,355 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:24:32,356 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:24:32,356 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:24:32,356 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:24:32,368 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36693-0x10015be5b1a0000, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-11T16:24:32,368 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42595-0x10015be5b1a0002, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/7446a65196a1,42595,1733934228675
2024-12-11T16:24:32,368 INFO  [RS:1;7446a65196a1:42595 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-11T16:24:32,369 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [7446a65196a1,42595,1733934228675]
2024-12-11T16:24:32,371 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:24:32,371 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:24:32,385 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/7446a65196a1,42595,1733934228675 already deleted, retry=false
2024-12-11T16:24:32,385 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 7446a65196a1,42595,1733934228675 expired; onlineServers=1
2024-12-11T16:24:32,477 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42595-0x10015be5b1a0002, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-11T16:24:32,477 INFO  [RS:1;7446a65196a1:42595 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-11T16:24:32,477 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42595-0x10015be5b1a0002, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-11T16:24:32,477 INFO  [RS:1;7446a65196a1:42595 {}] regionserver.HRegionServer(1031): Exiting; stopping=7446a65196a1,42595,1733934228675; zookeeper connection closed.
2024-12-11T16:24:32,477 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@b022db1 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@b022db1
2024-12-11T16:24:32,874 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-11T16:24:32,889 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:24:32,889 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:24:32,889 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:24:32,891 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:24:32,891 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:24:32,891 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:24:32,895 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:24:32,898 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:24:32,903 ERROR [WAL-Shutdown-0 {}] wal.AbstractFSWAL(2118): We have waited 5 seconds but the close of async writer doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.fshlog.wait.on.shutdown.seconds"
2024-12-11T16:24:32,906 DEBUG [RS:0;7446a65196a1:45947 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/oldWALs
2024-12-11T16:24:32,906 INFO  [RS:0;7446a65196a1:45947 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 7446a65196a1%2C45947%2C1733934227248.meta:.meta(num 1733934267301)
2024-12-11T16:24:32,906 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:24:32,906 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:24:32,907 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:24:32,907 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:24:32,908 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:24:32,910 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741880_1064 (size=18156)
2024-12-11T16:24:32,910 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40471 is added to blk_1073741880_1064 (size=18156)
2024-12-11T16:24:32,912 DEBUG [RS:0;7446a65196a1:45947 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/oldWALs
2024-12-11T16:24:32,912 INFO  [RS:0;7446a65196a1:45947 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 7446a65196a1%2C45947%2C1733934227248:(num 1733934266599)
2024-12-11T16:24:32,912 DEBUG [RS:0;7446a65196a1:45947 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T16:24:32,912 INFO  [RS:0;7446a65196a1:45947 {}] regionserver.LeaseManager(133): Closed leases
2024-12-11T16:24:32,912 INFO  [RS:0;7446a65196a1:45947 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-11T16:24:32,912 INFO  [RS:0;7446a65196a1:45947 {}] hbase.ChoreService(370): Chore service for: regionserver/7446a65196a1:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-12-11T16:24:32,912 INFO  [RS:0;7446a65196a1:45947 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-11T16:24:32,912 INFO  [regionserver/7446a65196a1:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-11T16:24:32,912 INFO  [RS:0;7446a65196a1:45947 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45947
2024-12-11T16:24:32,960 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45947-0x10015be5b1a0001, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/7446a65196a1,45947,1733934227248
2024-12-11T16:24:32,960 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36693-0x10015be5b1a0000, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-11T16:24:32,960 INFO  [RS:0;7446a65196a1:45947 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-11T16:24:32,968 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [7446a65196a1,45947,1733934227248]
2024-12-11T16:24:32,976 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/7446a65196a1,45947,1733934227248 already deleted, retry=false
2024-12-11T16:24:32,976 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 7446a65196a1,45947,1733934227248 expired; onlineServers=0
2024-12-11T16:24:32,976 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '7446a65196a1,36693,1733934227006' *****
2024-12-11T16:24:32,976 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-12-11T16:24:32,977 INFO  [M:0;7446a65196a1:36693 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-11T16:24:32,977 INFO  [M:0;7446a65196a1:36693 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-11T16:24:32,977 DEBUG [M:0;7446a65196a1:36693 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-12-11T16:24:32,977 DEBUG [M:0;7446a65196a1:36693 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-12-11T16:24:32,977 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-12-11T16:24:32,977 DEBUG [master/7446a65196a1:0:becomeActiveMaster-HFileCleaner.large.0-1733934227659 {}] cleaner.HFileCleaner(306): Exit Thread[master/7446a65196a1:0:becomeActiveMaster-HFileCleaner.large.0-1733934227659,5,FailOnTimeoutGroup]
2024-12-11T16:24:32,977 DEBUG [master/7446a65196a1:0:becomeActiveMaster-HFileCleaner.small.0-1733934227659 {}] cleaner.HFileCleaner(306): Exit Thread[master/7446a65196a1:0:becomeActiveMaster-HFileCleaner.small.0-1733934227659,5,FailOnTimeoutGroup]
2024-12-11T16:24:32,977 INFO  [M:0;7446a65196a1:36693 {}] hbase.ChoreService(370): Chore service for: master/7446a65196a1:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-12-11T16:24:32,977 INFO  [M:0;7446a65196a1:36693 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-11T16:24:32,977 DEBUG [M:0;7446a65196a1:36693 {}] master.HMaster(1795): Stopping service threads
2024-12-11T16:24:32,978 INFO  [M:0;7446a65196a1:36693 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-12-11T16:24:32,978 INFO  [M:0;7446a65196a1:36693 {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-11T16:24:32,978 INFO  [M:0;7446a65196a1:36693 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-12-11T16:24:32,978 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-12-11T16:24:32,985 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36693-0x10015be5b1a0000, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-12-11T16:24:32,985 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36693-0x10015be5b1a0000, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:24:32,985 DEBUG [M:0;7446a65196a1:36693 {}] zookeeper.ZKUtil(347): master:36693-0x10015be5b1a0000, quorum=127.0.0.1:53294, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-12-11T16:24:32,985 WARN  [M:0;7446a65196a1:36693 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-12-11T16:24:32,986 INFO  [M:0;7446a65196a1:36693 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/.lastflushedseqids
2024-12-11T16:24:32,996 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741895_1081 (size=130)
2024-12-11T16:24:32,996 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40471 is added to blk_1073741895_1081 (size=130)
2024-12-11T16:24:32,997 INFO  [M:0;7446a65196a1:36693 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-12-11T16:24:32,997 INFO  [M:0;7446a65196a1:36693 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-12-11T16:24:32,997 DEBUG [M:0;7446a65196a1:36693 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-11T16:24:32,997 INFO  [M:0;7446a65196a1:36693 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T16:24:32,997 DEBUG [M:0;7446a65196a1:36693 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T16:24:32,997 DEBUG [M:0;7446a65196a1:36693 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-11T16:24:32,997 DEBUG [M:0;7446a65196a1:36693 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T16:24:32,997 INFO  [M:0;7446a65196a1:36693 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.25 KB heapSize=29.49 KB
2024-12-11T16:24:33,015 DEBUG [M:0;7446a65196a1:36693 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8bb1939501324dfeab5b30cd8db20265 is 82, key is hbase:meta,,1/info:regioninfo/1733934228436/Put/seqid=0
2024-12-11T16:24:33,020 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741896_1082 (size=5672)
2024-12-11T16:24:33,020 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40471 is added to blk_1073741896_1082 (size=5672)
2024-12-11T16:24:33,021 INFO  [M:0;7446a65196a1:36693 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8bb1939501324dfeab5b30cd8db20265
2024-12-11T16:24:33,040 DEBUG [M:0;7446a65196a1:36693 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/cf11dce599a6404ca4c0e2840a86f285 is 774, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733934229350/Put/seqid=0
2024-12-11T16:24:33,045 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741897_1083 (size=6255)
2024-12-11T16:24:33,045 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40471 is added to blk_1073741897_1083 (size=6255)
2024-12-11T16:24:33,045 INFO  [M:0;7446a65196a1:36693 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.58 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/cf11dce599a6404ca4c0e2840a86f285
2024-12-11T16:24:33,050 INFO  [M:0;7446a65196a1:36693 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for cf11dce599a6404ca4c0e2840a86f285
2024-12-11T16:24:33,063 DEBUG [M:0;7446a65196a1:36693 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/6bdf3ea4e07647cf8385ef61693d9f86 is 69, key is 7446a65196a1,42595,1733934228675/rs:state/1733934228847/Put/seqid=0
2024-12-11T16:24:33,068 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45947-0x10015be5b1a0001, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-11T16:24:33,068 INFO  [RS:0;7446a65196a1:45947 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-11T16:24:33,068 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45947-0x10015be5b1a0001, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-11T16:24:33,068 INFO  [RS:0;7446a65196a1:45947 {}] regionserver.HRegionServer(1031): Exiting; stopping=7446a65196a1,45947,1733934227248; zookeeper connection closed.
2024-12-11T16:24:33,070 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@eb2ac81 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@eb2ac81
2024-12-11T16:24:33,070 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete
2024-12-11T16:24:33,071 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741898_1084 (size=5224)
2024-12-11T16:24:33,071 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40471 is added to blk_1073741898_1084 (size=5224)
2024-12-11T16:24:33,072 INFO  [M:0;7446a65196a1:36693 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/6bdf3ea4e07647cf8385ef61693d9f86
2024-12-11T16:24:33,091 DEBUG [M:0;7446a65196a1:36693 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/70428643581b4c6ca09c5b683349d4b3 is 52, key is load_balancer_on/state:d/1733934228650/Put/seqid=0
2024-12-11T16:24:33,096 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741899_1085 (size=5056)
2024-12-11T16:24:33,097 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40471 is added to blk_1073741899_1085 (size=5056)
2024-12-11T16:24:33,097 INFO  [M:0;7446a65196a1:36693 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/70428643581b4c6ca09c5b683349d4b3
2024-12-11T16:24:33,103 DEBUG [M:0;7446a65196a1:36693 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8bb1939501324dfeab5b30cd8db20265 as hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/8bb1939501324dfeab5b30cd8db20265
2024-12-11T16:24:33,109 INFO  [M:0;7446a65196a1:36693 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/8bb1939501324dfeab5b30cd8db20265, entries=8, sequenceid=60, filesize=5.5 K
2024-12-11T16:24:33,111 DEBUG [M:0;7446a65196a1:36693 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/cf11dce599a6404ca4c0e2840a86f285 as hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/cf11dce599a6404ca4c0e2840a86f285
2024-12-11T16:24:33,117 INFO  [M:0;7446a65196a1:36693 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for cf11dce599a6404ca4c0e2840a86f285
2024-12-11T16:24:33,117 INFO  [M:0;7446a65196a1:36693 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/cf11dce599a6404ca4c0e2840a86f285, entries=6, sequenceid=60, filesize=6.1 K
2024-12-11T16:24:33,118 DEBUG [M:0;7446a65196a1:36693 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/6bdf3ea4e07647cf8385ef61693d9f86 as hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/6bdf3ea4e07647cf8385ef61693d9f86
2024-12-11T16:24:33,126 INFO  [M:0;7446a65196a1:36693 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/6bdf3ea4e07647cf8385ef61693d9f86, entries=2, sequenceid=60, filesize=5.1 K
2024-12-11T16:24:33,127 DEBUG [M:0;7446a65196a1:36693 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/70428643581b4c6ca09c5b683349d4b3 as hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/70428643581b4c6ca09c5b683349d4b3
2024-12-11T16:24:33,132 INFO  [M:0;7446a65196a1:36693 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/70428643581b4c6ca09c5b683349d4b3, entries=1, sequenceid=60, filesize=4.9 K
2024-12-11T16:24:33,134 INFO  [M:0;7446a65196a1:36693 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.25 KB/23805, heapSize ~29.43 KB/30136, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 136ms, sequenceid=60, compaction requested=false
2024-12-11T16:24:33,139 INFO  [M:0;7446a65196a1:36693 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T16:24:33,139 DEBUG [M:0;7446a65196a1:36693 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733934272997Disabling compacts and flushes for region at 1733934272997Disabling writes for close at 1733934272997Obtaining lock to block concurrent updates at 1733934272997Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733934272997Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23805, getHeapSize=30136, getOffHeapSize=0, getCellsCount=71 at 1733934272998 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733934272999 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733934272999Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733934273015 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733934273015Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733934273026 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733934273039 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733934273039Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733934273050 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733934273063 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733934273063Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733934273077 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733934273091 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733934273091Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@a654c76: reopening flushed file at 1733934273102 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@65e85886: reopening flushed file at 1733934273110 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@117aca36: reopening flushed file at 1733934273117 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@c693b27: reopening flushed file at 1733934273126 (+9 ms)Finished flush of dataSize ~23.25 KB/23805, heapSize ~29.43 KB/30136, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 136ms, sequenceid=60, compaction requested=false at 1733934273134 (+8 ms)Writing region close event to WAL at 1733934273139 (+5 ms)Closed at 1733934273139
2024-12-11T16:24:33,139 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:24:33,139 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:24:33,139 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:24:33,140 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:24:33,140 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:24:33,141 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40471 is added to blk_1073741878_1061 (size=1045)
2024-12-11T16:24:33,142 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741878_1061 (size=1045)
2024-12-11T16:24:33,143 INFO  [M:0;7446a65196a1:36693 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-12-11T16:24:33,143 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-11T16:24:33,143 INFO  [M:0;7446a65196a1:36693 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36693
2024-12-11T16:24:33,144 INFO  [M:0;7446a65196a1:36693 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-11T16:24:33,266 INFO  [M:0;7446a65196a1:36693 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-11T16:24:33,266 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36693-0x10015be5b1a0000, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-11T16:24:33,266 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36693-0x10015be5b1a0000, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-11T16:24:33,269 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7cac94da{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T16:24:33,270 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@894b118{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-11T16:24:33,270 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-11T16:24:33,270 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@758da55c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-11T16:24:33,270 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3955e9ae{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/hadoop.log.dir/,STOPPED}
2024-12-11T16:24:33,271 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-11T16:24:33,271 WARN  [BP-1552246360-172.17.0.2-1733934224410 heartbeating to localhost/127.0.0.1:34525 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-11T16:24:33,271 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-11T16:24:33,271 WARN  [BP-1552246360-172.17.0.2-1733934224410 heartbeating to localhost/127.0.0.1:34525 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1552246360-172.17.0.2-1733934224410 (Datanode Uuid 3250d228-4a4c-40f3-8d16-2b51b2a499ea) service to localhost/127.0.0.1:34525
2024-12-11T16:24:33,271 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@5e2bf7d5 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1552246360-172.17.0.2-1733934224410:blk_1073741837_1013, datanode=DatanodeInfoWithStorage[127.0.0.1:44567,null,null])
java.io.InterruptedIOException: DestHost:destPort localhost:38073 , LocalHost:localPort 7446a65196a1/172.17.0.2:0. Failed on local exception: java.io.InterruptedIOException: Interrupted: action=RetryAction(action=RETRY, delayMillis=1000, reason=retries get failed due to exceeded maximum allowed retries number: 10), retry policy=RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS)
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:936) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.InterruptedIOException: Interrupted: action=RetryAction(action=RETRY, delayMillis=1000, reason=retries get failed due to exceeded maximum allowed retries number: 10), retry policy=RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS)
	at org.apache.hadoop.ipc.Client$Connection.handleConnectionFailure(Client.java:963) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:691) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
Caused by: java.lang.InterruptedException: sleep interrupted
	at java.lang.Thread.sleep(Native Method) ~[?:?]
	at org.apache.hadoop.ipc.Client$Connection.handleConnectionFailure(Client.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:691) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
2024-12-11T16:24:33,271 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@5e2bf7d5 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1552246360-172.17.0.2-1733934224410:blk_1073741837_1013, datanode=DatanodeInfoWithStorage[127.0.0.1:43417,null,null])
java.io.IOException: No block pool offer service for bpid=BP-1552246360-172.17.0.2-1733934224410
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.getDatanodeID(BlockRecoveryWorker.java:539) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$000(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:131) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T16:24:33,272 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@5e2bf7d5 {}] datanode.BlockRecoveryWorker$1(605): recover Block: RecoveringBlock{BP-1552246360-172.17.0.2-1733934224410:blk_1073741837_1013; getBlockSize()=85; corrupt=false; offset=-1; locs=[DatanodeInfoWithStorage[127.0.0.1:44567,null,null], DatanodeInfoWithStorage[127.0.0.1:43417,null,null]]; cachedLocs=[]} FAILED: java.io.IOException: All datanodes failed: block=BP-1552246360-172.17.0.2-1733934224410:blk_1073741837_1013, datanodeids=[DatanodeInfoWithStorage[127.0.0.1:44567,null,null], DatanodeInfoWithStorage[127.0.0.1:43417,null,null]]
2024-12-11T16:24:33,272 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@5e2bf7d5 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1552246360-172.17.0.2-1733934224410:blk_1073741834_1010, datanode=DatanodeInfoWithStorage[127.0.0.1:43417,null,null])
java.io.IOException: No block pool offer service for bpid=BP-1552246360-172.17.0.2-1733934224410
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.getDatanodeID(BlockRecoveryWorker.java:539) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$000(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:131) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T16:24:33,272 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/cluster_7052d23d-6944-2d6a-b15d-561f403fc1d1/data/data3/current/BP-1552246360-172.17.0.2-1733934224410 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T16:24:33,272 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@5e2bf7d5 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1552246360-172.17.0.2-1733934224410:blk_1073741834_1010, datanode=DatanodeInfoWithStorage[127.0.0.1:44567,null,null])
java.io.IOException: No block pool offer service for bpid=BP-1552246360-172.17.0.2-1733934224410
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.getDatanodeID(BlockRecoveryWorker.java:539) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$000(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:131) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T16:24:33,272 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@5e2bf7d5 {}] datanode.BlockRecoveryWorker$1(605): recover Block: RecoveringBlock{BP-1552246360-172.17.0.2-1733934224410:blk_1073741834_1010; getBlockSize()=85; corrupt=false; offset=-1; locs=[DatanodeInfoWithStorage[127.0.0.1:43417,null,null], DatanodeInfoWithStorage[127.0.0.1:44567,null,null]]; cachedLocs=[]} FAILED: java.io.IOException: All datanodes failed: block=BP-1552246360-172.17.0.2-1733934224410:blk_1073741834_1010, datanodeids=[DatanodeInfoWithStorage[127.0.0.1:43417,null,null], DatanodeInfoWithStorage[127.0.0.1:44567,null,null]]
2024-12-11T16:24:33,272 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/cluster_7052d23d-6944-2d6a-b15d-561f403fc1d1/data/data4/current/BP-1552246360-172.17.0.2-1733934224410 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T16:24:33,272 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-11T16:24:33,275 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7424ce2{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T16:24:33,275 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7929c9ee{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-11T16:24:33,275 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-11T16:24:33,275 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@75748ead{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-11T16:24:33,275 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4136d603{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/hadoop.log.dir/,STOPPED}
2024-12-11T16:24:33,278 WARN  [BP-1552246360-172.17.0.2-1733934224410 heartbeating to localhost/127.0.0.1:34525 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-11T16:24:33,278 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-11T16:24:33,278 WARN  [BP-1552246360-172.17.0.2-1733934224410 heartbeating to localhost/127.0.0.1:34525 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1552246360-172.17.0.2-1733934224410 (Datanode Uuid 852f5a8b-0ced-4a59-b572-6ce7ea337674) service to localhost/127.0.0.1:34525
2024-12-11T16:24:33,278 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-11T16:24:33,279 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/cluster_7052d23d-6944-2d6a-b15d-561f403fc1d1/data/data9/current/BP-1552246360-172.17.0.2-1733934224410 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T16:24:33,279 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/cluster_7052d23d-6944-2d6a-b15d-561f403fc1d1/data/data10/current/BP-1552246360-172.17.0.2-1733934224410 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T16:24:33,280 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-11T16:24:33,287 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6668e7cf{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-11T16:24:33,288 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@a936cd8{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-11T16:24:33,288 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-11T16:24:33,288 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@c0b5dad{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-11T16:24:33,288 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@f31077{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/hadoop.log.dir/,STOPPED}
2024-12-11T16:24:33,297 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-12-11T16:24:33,304 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:24:33,316 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:24:33,331 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-12-11T16:24:33,340 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnDatanodeDeath Thread=160 (was 81)
Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-9
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-7-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (548703531) connection to localhost/127.0.0.1:34525 from jenkins.hfs.2
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-19-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:34525
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:34957
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-5-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-21-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-19-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-7
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-7-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-5-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-6
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Close-WAL-Writer-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:175)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$893/0x00007fd98cbf46e8.run(Unknown Source)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.3@localhost:34525
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-19-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (548703531) connection to localhost/127.0.0.1:34525 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: ForkJoinPool-2-worker-5
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: nioEventLoopGroup-18-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Close-WAL-Writer-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:175)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$893/0x00007fd98cbf46e8.run(Unknown Source)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-7-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:34525
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:34525
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (548703531) connection to localhost/127.0.0.1:34525 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-21-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.1@localhost:34957
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Timer for 'DataNode' metrics system
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-18-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.2@localhost:34525
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-20-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:34525
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-8
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-20-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-5-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-18-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:34525
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (548703531) connection to localhost/127.0.0.1:34525 from jenkins.hfs.3
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-21-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-20-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-5
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=452 (was 404) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=638 (was 893), ProcessCount=11 (was 11), AvailableMemoryMB=845 (was 1330)
2024-12-11T16:24:33,347 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnPipelineRestart Thread=160, OpenFileDescriptor=452, MaxFileDescriptor=1048576, SystemLoadAverage=638, ProcessCount=11, AvailableMemoryMB=845
2024-12-11T16:24:33,347 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-12-11T16:24:33,347 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/hadoop.log.dir so I do NOT create it in target/test-data/c90c660c-600a-9166-c37d-1b9c62c53b37
2024-12-11T16:24:33,347 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/53036128-f216-6ce6-44c0-30b05c226f86/hadoop.tmp.dir so I do NOT create it in target/test-data/c90c660c-600a-9166-c37d-1b9c62c53b37
2024-12-11T16:24:33,347 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90c660c-600a-9166-c37d-1b9c62c53b37/cluster_c6aa3bec-caf4-e30f-1389-b14fd52f58cf, deleteOnExit=true
2024-12-11T16:24:33,347 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-12-11T16:24:33,348 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90c660c-600a-9166-c37d-1b9c62c53b37/test.cache.data in system properties and HBase conf
2024-12-11T16:24:33,348 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90c660c-600a-9166-c37d-1b9c62c53b37/hadoop.tmp.dir in system properties and HBase conf
2024-12-11T16:24:33,348 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90c660c-600a-9166-c37d-1b9c62c53b37/hadoop.log.dir in system properties and HBase conf
2024-12-11T16:24:33,348 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90c660c-600a-9166-c37d-1b9c62c53b37/mapreduce.cluster.local.dir in system properties and HBase conf
2024-12-11T16:24:33,348 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90c660c-600a-9166-c37d-1b9c62c53b37/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-12-11T16:24:33,348 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-12-11T16:24:33,348 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-11T16:24:33,348 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90c660c-600a-9166-c37d-1b9c62c53b37/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-12-11T16:24:33,348 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90c660c-600a-9166-c37d-1b9c62c53b37/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-12-11T16:24:33,348 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90c660c-600a-9166-c37d-1b9c62c53b37/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-12-11T16:24:33,348 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90c660c-600a-9166-c37d-1b9c62c53b37/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-11T16:24:33,348 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90c660c-600a-9166-c37d-1b9c62c53b37/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-12-11T16:24:33,348 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90c660c-600a-9166-c37d-1b9c62c53b37/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-12-11T16:24:33,349 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90c660c-600a-9166-c37d-1b9c62c53b37/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-11T16:24:33,349 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90c660c-600a-9166-c37d-1b9c62c53b37/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-11T16:24:33,349 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90c660c-600a-9166-c37d-1b9c62c53b37/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-12-11T16:24:33,349 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90c660c-600a-9166-c37d-1b9c62c53b37/nfs.dump.dir in system properties and HBase conf
2024-12-11T16:24:33,349 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90c660c-600a-9166-c37d-1b9c62c53b37/java.io.tmpdir in system properties and HBase conf
2024-12-11T16:24:33,349 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90c660c-600a-9166-c37d-1b9c62c53b37/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-11T16:24:33,349 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90c660c-600a-9166-c37d-1b9c62c53b37/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-12-11T16:24:33,349 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90c660c-600a-9166-c37d-1b9c62c53b37/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-12-11T16:24:33,360 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-11T16:24:33,584 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-11T16:24:33,588 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-11T16:24:33,589 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-11T16:24:33,589 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-11T16:24:33,589 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-11T16:24:33,590 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-11T16:24:33,590 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@43b12dab{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90c660c-600a-9166-c37d-1b9c62c53b37/hadoop.log.dir/,AVAILABLE}
2024-12-11T16:24:33,591 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@cdcce9d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-11T16:24:33,684 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7a25f841{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90c660c-600a-9166-c37d-1b9c62c53b37/java.io.tmpdir/jetty-localhost-38317-hadoop-hdfs-3_4_1-tests_jar-_-any-18289423757634141678/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-11T16:24:33,684 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@58a42b34{HTTP/1.1, (http/1.1)}{localhost:38317}
2024-12-11T16:24:33,684 INFO  [Time-limited test {}] server.Server(415): Started @164355ms
2024-12-11T16:24:33,694 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-11T16:24:33,910 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-11T16:24:33,914 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-11T16:24:33,914 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-11T16:24:33,915 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-11T16:24:33,915 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-11T16:24:33,915 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@520d4f99{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90c660c-600a-9166-c37d-1b9c62c53b37/hadoop.log.dir/,AVAILABLE}
2024-12-11T16:24:33,916 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5a617dcf{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-11T16:24:34,019 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@124f9706{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90c660c-600a-9166-c37d-1b9c62c53b37/java.io.tmpdir/jetty-localhost-39451-hadoop-hdfs-3_4_1-tests_jar-_-any-8512998316758071451/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T16:24:34,020 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@52bf6dde{HTTP/1.1, (http/1.1)}{localhost:39451}
2024-12-11T16:24:34,020 INFO  [Time-limited test {}] server.Server(415): Started @164691ms
2024-12-11T16:24:34,021 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-11T16:24:34,046 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-11T16:24:34,049 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-11T16:24:34,050 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-11T16:24:34,050 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-11T16:24:34,050 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-11T16:24:34,050 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3a6fb45d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90c660c-600a-9166-c37d-1b9c62c53b37/hadoop.log.dir/,AVAILABLE}
2024-12-11T16:24:34,051 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@61ecc897{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-11T16:24:34,150 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@30a95171{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90c660c-600a-9166-c37d-1b9c62c53b37/java.io.tmpdir/jetty-localhost-36075-hadoop-hdfs-3_4_1-tests_jar-_-any-3187496746330215990/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T16:24:34,150 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4586c70{HTTP/1.1, (http/1.1)}{localhost:36075}
2024-12-11T16:24:34,150 INFO  [Time-limited test {}] server.Server(415): Started @164820ms
2024-12-11T16:24:34,151 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-11T16:24:34,305 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:24:34,317 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:24:34,883 WARN  [Thread-1208 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90c660c-600a-9166-c37d-1b9c62c53b37/cluster_c6aa3bec-caf4-e30f-1389-b14fd52f58cf/data/data1/current/BP-119074730-172.17.0.2-1733934273371/current, will proceed with Du for space computation calculation, 
2024-12-11T16:24:34,883 WARN  [Thread-1209 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90c660c-600a-9166-c37d-1b9c62c53b37/cluster_c6aa3bec-caf4-e30f-1389-b14fd52f58cf/data/data2/current/BP-119074730-172.17.0.2-1733934273371/current, will proceed with Du for space computation calculation, 
2024-12-11T16:24:34,897 WARN  [Thread-1172 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-11T16:24:34,899 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x7fbf746153cdb3cb with lease ID 0x5a0312432928716b: Processing first storage report for DS-d0922336-6ec8-41bd-9084-f220c6cd95f5 from datanode DatanodeRegistration(127.0.0.1:43085, datanodeUuid=94f6fc17-0a75-416c-8864-c6b69c7ce840, infoPort=34645, infoSecurePort=0, ipcPort=45759, storageInfo=lv=-57;cid=testClusterID;nsid=6436803;c=1733934273371)
2024-12-11T16:24:34,900 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x7fbf746153cdb3cb with lease ID 0x5a0312432928716b: from storage DS-d0922336-6ec8-41bd-9084-f220c6cd95f5 node DatanodeRegistration(127.0.0.1:43085, datanodeUuid=94f6fc17-0a75-416c-8864-c6b69c7ce840, infoPort=34645, infoSecurePort=0, ipcPort=45759, storageInfo=lv=-57;cid=testClusterID;nsid=6436803;c=1733934273371), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-11T16:24:34,900 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x7fbf746153cdb3cb with lease ID 0x5a0312432928716b: Processing first storage report for DS-eb41716c-cf35-4f8a-a3bd-166a79ec6316 from datanode DatanodeRegistration(127.0.0.1:43085, datanodeUuid=94f6fc17-0a75-416c-8864-c6b69c7ce840, infoPort=34645, infoSecurePort=0, ipcPort=45759, storageInfo=lv=-57;cid=testClusterID;nsid=6436803;c=1733934273371)
2024-12-11T16:24:34,900 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x7fbf746153cdb3cb with lease ID 0x5a0312432928716b: from storage DS-eb41716c-cf35-4f8a-a3bd-166a79ec6316 node DatanodeRegistration(127.0.0.1:43085, datanodeUuid=94f6fc17-0a75-416c-8864-c6b69c7ce840, infoPort=34645, infoSecurePort=0, ipcPort=45759, storageInfo=lv=-57;cid=testClusterID;nsid=6436803;c=1733934273371), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-12-11T16:24:35,010 WARN  [Thread-1220 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90c660c-600a-9166-c37d-1b9c62c53b37/cluster_c6aa3bec-caf4-e30f-1389-b14fd52f58cf/data/data4/current/BP-119074730-172.17.0.2-1733934273371/current, will proceed with Du for space computation calculation, 
2024-12-11T16:24:35,010 WARN  [Thread-1219 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90c660c-600a-9166-c37d-1b9c62c53b37/cluster_c6aa3bec-caf4-e30f-1389-b14fd52f58cf/data/data3/current/BP-119074730-172.17.0.2-1733934273371/current, will proceed with Du for space computation calculation, 
2024-12-11T16:24:35,030 WARN  [Thread-1195 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-11T16:24:35,032 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe0412879afc4e182 with lease ID 0x5a0312432928716c: Processing first storage report for DS-31d80686-d824-4918-894d-bf029f453b89 from datanode DatanodeRegistration(127.0.0.1:37795, datanodeUuid=518c53e3-a44d-421f-a0df-e28575189a25, infoPort=35247, infoSecurePort=0, ipcPort=41861, storageInfo=lv=-57;cid=testClusterID;nsid=6436803;c=1733934273371)
2024-12-11T16:24:35,032 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe0412879afc4e182 with lease ID 0x5a0312432928716c: from storage DS-31d80686-d824-4918-894d-bf029f453b89 node DatanodeRegistration(127.0.0.1:37795, datanodeUuid=518c53e3-a44d-421f-a0df-e28575189a25, infoPort=35247, infoSecurePort=0, ipcPort=41861, storageInfo=lv=-57;cid=testClusterID;nsid=6436803;c=1733934273371), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-11T16:24:35,032 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe0412879afc4e182 with lease ID 0x5a0312432928716c: Processing first storage report for DS-008cd667-4edc-4c90-92f3-4613f09bac93 from datanode DatanodeRegistration(127.0.0.1:37795, datanodeUuid=518c53e3-a44d-421f-a0df-e28575189a25, infoPort=35247, infoSecurePort=0, ipcPort=41861, storageInfo=lv=-57;cid=testClusterID;nsid=6436803;c=1733934273371)
2024-12-11T16:24:35,032 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe0412879afc4e182 with lease ID 0x5a0312432928716c: from storage DS-008cd667-4edc-4c90-92f3-4613f09bac93 node DatanodeRegistration(127.0.0.1:37795, datanodeUuid=518c53e3-a44d-421f-a0df-e28575189a25, infoPort=35247, infoSecurePort=0, ipcPort=41861, storageInfo=lv=-57;cid=testClusterID;nsid=6436803;c=1733934273371), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-11T16:24:35,088 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90c660c-600a-9166-c37d-1b9c62c53b37
2024-12-11T16:24:35,090 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90c660c-600a-9166-c37d-1b9c62c53b37/cluster_c6aa3bec-caf4-e30f-1389-b14fd52f58cf/zookeeper_0, clientPort=50451, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90c660c-600a-9166-c37d-1b9c62c53b37/cluster_c6aa3bec-caf4-e30f-1389-b14fd52f58cf/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90c660c-600a-9166-c37d-1b9c62c53b37/cluster_c6aa3bec-caf4-e30f-1389-b14fd52f58cf/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-11T16:24:35,091 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=50451
2024-12-11T16:24:35,092 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T16:24:35,093 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T16:24:35,103 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43085 is added to blk_1073741825_1001 (size=7)
2024-12-11T16:24:35,103 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37795 is added to blk_1073741825_1001 (size=7)
2024-12-11T16:24:35,104 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1 with version=8
2024-12-11T16:24:35,104 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/hbase-staging
2024-12-11T16:24:35,106 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/7446a65196a1:0 server-side Connection retries=45
2024-12-11T16:24:35,106 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-11T16:24:35,106 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-11T16:24:35,107 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-11T16:24:35,107 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-11T16:24:35,107 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-11T16:24:35,107 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-12-11T16:24:35,107 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-11T16:24:35,108 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37355
2024-12-11T16:24:35,109 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:37355 connecting to ZooKeeper ensemble=127.0.0.1:50451
2024-12-11T16:24:35,159 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:373550x0, quorum=127.0.0.1:50451, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-11T16:24:35,160 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:37355-0x10015bf18ae0000 connected
2024-12-11T16:24:35,227 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T16:24:35,231 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T16:24:35,235 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37355-0x10015bf18ae0000, quorum=127.0.0.1:50451, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-11T16:24:35,235 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1, hbase.cluster.distributed=false
2024-12-11T16:24:35,237 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37355-0x10015bf18ae0000, quorum=127.0.0.1:50451, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-11T16:24:35,237 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37355
2024-12-11T16:24:35,238 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37355
2024-12-11T16:24:35,238 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37355
2024-12-11T16:24:35,238 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37355
2024-12-11T16:24:35,238 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37355
2024-12-11T16:24:35,251 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/7446a65196a1:0 server-side Connection retries=45
2024-12-11T16:24:35,251 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-11T16:24:35,251 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-11T16:24:35,251 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-11T16:24:35,251 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-11T16:24:35,251 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-11T16:24:35,251 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-11T16:24:35,251 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-11T16:24:35,252 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45491
2024-12-11T16:24:35,253 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45491 connecting to ZooKeeper ensemble=127.0.0.1:50451
2024-12-11T16:24:35,253 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T16:24:35,254 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T16:24:35,268 DEBUG [pool-525-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:454910x0, quorum=127.0.0.1:50451, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-11T16:24:35,268 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45491-0x10015bf18ae0001 connected
2024-12-11T16:24:35,268 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45491-0x10015bf18ae0001, quorum=127.0.0.1:50451, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-11T16:24:35,269 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-11T16:24:35,269 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-11T16:24:35,270 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45491-0x10015bf18ae0001, quorum=127.0.0.1:50451, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-11T16:24:35,271 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45491-0x10015bf18ae0001, quorum=127.0.0.1:50451, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-11T16:24:35,274 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45491
2024-12-11T16:24:35,274 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45491
2024-12-11T16:24:35,274 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45491
2024-12-11T16:24:35,274 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45491
2024-12-11T16:24:35,274 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45491
2024-12-11T16:24:35,285 DEBUG [M:0;7446a65196a1:37355 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;7446a65196a1:37355
2024-12-11T16:24:35,285 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/7446a65196a1,37355,1733934275106
2024-12-11T16:24:35,292 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37355-0x10015bf18ae0000, quorum=127.0.0.1:50451, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-11T16:24:35,292 DEBUG [pool-525-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45491-0x10015bf18ae0001, quorum=127.0.0.1:50451, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-11T16:24:35,293 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37355-0x10015bf18ae0000, quorum=127.0.0.1:50451, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/7446a65196a1,37355,1733934275106
2024-12-11T16:24:35,301 DEBUG [pool-525-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45491-0x10015bf18ae0001, quorum=127.0.0.1:50451, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-11T16:24:35,301 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37355-0x10015bf18ae0000, quorum=127.0.0.1:50451, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:24:35,301 DEBUG [pool-525-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45491-0x10015bf18ae0001, quorum=127.0.0.1:50451, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:24:35,301 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37355-0x10015bf18ae0000, quorum=127.0.0.1:50451, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-11T16:24:35,302 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/7446a65196a1,37355,1733934275106 from backup master directory
2024-12-11T16:24:35,306 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:24:35,309 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37355-0x10015bf18ae0000, quorum=127.0.0.1:50451, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/7446a65196a1,37355,1733934275106
2024-12-11T16:24:35,309 DEBUG [pool-525-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45491-0x10015bf18ae0001, quorum=127.0.0.1:50451, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-11T16:24:35,309 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37355-0x10015bf18ae0000, quorum=127.0.0.1:50451, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-11T16:24:35,309 WARN  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-11T16:24:35,309 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=7446a65196a1,37355,1733934275106
2024-12-11T16:24:35,316 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/hbase.id] with ID: a6d37003-9a9d-410f-8ed6-83d51fe14411
2024-12-11T16:24:35,316 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/.tmp/hbase.id
2024-12-11T16:24:35,318 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:24:35,325 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37795 is added to blk_1073741826_1002 (size=42)
2024-12-11T16:24:35,325 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43085 is added to blk_1073741826_1002 (size=42)
2024-12-11T16:24:35,325 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/.tmp/hbase.id]:[hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/hbase.id]
2024-12-11T16:24:35,339 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T16:24:35,339 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-12-11T16:24:35,340 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms.
2024-12-11T16:24:35,351 DEBUG [pool-525-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45491-0x10015bf18ae0001, quorum=127.0.0.1:50451, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:24:35,351 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37355-0x10015bf18ae0000, quorum=127.0.0.1:50451, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:24:35,357 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37795 is added to blk_1073741827_1003 (size=196)
2024-12-11T16:24:35,357 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43085 is added to blk_1073741827_1003 (size=196)
2024-12-11T16:24:35,358 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-11T16:24:35,359 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-11T16:24:35,359 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-11T16:24:35,376 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43085 is added to blk_1073741828_1004 (size=1189)
2024-12-11T16:24:35,376 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37795 is added to blk_1073741828_1004 (size=1189)
2024-12-11T16:24:35,380 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/MasterData/data/master/store
2024-12-11T16:24:35,391 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37795 is added to blk_1073741829_1005 (size=34)
2024-12-11T16:24:35,394 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43085 is added to blk_1073741829_1005 (size=34)
2024-12-11T16:24:35,395 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T16:24:35,395 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-11T16:24:35,395 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T16:24:35,395 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T16:24:35,395 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-11T16:24:35,395 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T16:24:35,395 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T16:24:35,396 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733934275395Disabling compacts and flushes for region at 1733934275395Disabling writes for close at 1733934275395Writing region close event to WAL at 1733934275395Closed at 1733934275395
2024-12-11T16:24:35,396 WARN  [master/7446a65196a1:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/MasterData/data/master/store/.initializing
2024-12-11T16:24:35,396 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/MasterData/WALs/7446a65196a1,37355,1733934275106
2024-12-11T16:24:35,401 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7446a65196a1%2C37355%2C1733934275106, suffix=, logDir=hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/MasterData/WALs/7446a65196a1,37355,1733934275106, archiveDir=hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/MasterData/oldWALs, maxLogs=10
2024-12-11T16:24:35,402 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7446a65196a1%2C37355%2C1733934275106.1733934275402
2024-12-11T16:24:35,409 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/MasterData/WALs/7446a65196a1,37355,1733934275106/7446a65196a1%2C37355%2C1733934275106.1733934275402
2024-12-11T16:24:35,410 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34645:34645),(127.0.0.1/127.0.0.1:35247:35247)]
2024-12-11T16:24:35,414 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-11T16:24:35,414 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T16:24:35,414 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T16:24:35,414 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T16:24:35,418 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-11T16:24:35,419 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-11T16:24:35,419 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:24:35,420 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T16:24:35,420 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-11T16:24:35,421 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-11T16:24:35,421 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:24:35,421 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-11T16:24:35,422 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-11T16:24:35,423 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-11T16:24:35,423 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:24:35,423 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-11T16:24:35,423 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-11T16:24:35,424 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-11T16:24:35,424 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:24:35,425 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-11T16:24:35,425 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T16:24:35,425 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-11T16:24:35,425 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-11T16:24:35,426 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T16:24:35,427 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T16:24:35,427 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-11T16:24:35,428 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T16:24:35,430 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-11T16:24:35,430 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=723157, jitterRate=-0.0804586261510849}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-11T16:24:35,431 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1733934275414Initializing all the Stores at 1733934275415 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733934275415Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733934275418 (+3 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733934275418Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733934275418Cleaning up temporary data from old regions at 1733934275427 (+9 ms)Region opened successfully at 1733934275431 (+4 ms)
2024-12-11T16:24:35,431 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-11T16:24:35,434 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8a6942, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7446a65196a1/172.17.0.2:0
2024-12-11T16:24:35,435 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-12-11T16:24:35,435 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-11T16:24:35,435 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-11T16:24:35,435 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-11T16:24:35,436 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-12-11T16:24:35,436 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-12-11T16:24:35,436 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-11T16:24:35,439 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-11T16:24:35,439 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37355-0x10015bf18ae0000, quorum=127.0.0.1:50451, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-11T16:24:35,467 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-12-11T16:24:35,468 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-11T16:24:35,469 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37355-0x10015bf18ae0000, quorum=127.0.0.1:50451, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-11T16:24:35,490 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-12-11T16:24:35,491 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-11T16:24:35,492 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37355-0x10015bf18ae0000, quorum=127.0.0.1:50451, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-11T16:24:35,501 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-12-11T16:24:35,502 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37355-0x10015bf18ae0000, quorum=127.0.0.1:50451, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-11T16:24:35,509 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-12-11T16:24:35,511 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37355-0x10015bf18ae0000, quorum=127.0.0.1:50451, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-11T16:24:35,517 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-11T16:24:35,526 DEBUG [pool-525-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45491-0x10015bf18ae0001, quorum=127.0.0.1:50451, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-11T16:24:35,526 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37355-0x10015bf18ae0000, quorum=127.0.0.1:50451, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-11T16:24:35,526 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37355-0x10015bf18ae0000, quorum=127.0.0.1:50451, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:24:35,526 DEBUG [pool-525-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45491-0x10015bf18ae0001, quorum=127.0.0.1:50451, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:24:35,527 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=7446a65196a1,37355,1733934275106, sessionid=0x10015bf18ae0000, setting cluster-up flag (Was=false)
2024-12-11T16:24:35,543 DEBUG [pool-525-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45491-0x10015bf18ae0001, quorum=127.0.0.1:50451, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:24:35,543 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37355-0x10015bf18ae0000, quorum=127.0.0.1:50451, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:24:35,567 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-11T16:24:35,569 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=7446a65196a1,37355,1733934275106
2024-12-11T16:24:35,584 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37355-0x10015bf18ae0000, quorum=127.0.0.1:50451, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:24:35,584 DEBUG [pool-525-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45491-0x10015bf18ae0001, quorum=127.0.0.1:50451, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:24:35,609 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-11T16:24:35,610 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=7446a65196a1,37355,1733934275106
2024-12-11T16:24:35,612 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-12-11T16:24:35,613 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-12-11T16:24:35,613 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-12-11T16:24:35,613 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-11T16:24:35,614 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 7446a65196a1,37355,1733934275106
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-11T16:24:35,615 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/7446a65196a1:0, corePoolSize=5, maxPoolSize=5
2024-12-11T16:24:35,615 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/7446a65196a1:0, corePoolSize=5, maxPoolSize=5
2024-12-11T16:24:35,615 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/7446a65196a1:0, corePoolSize=5, maxPoolSize=5
2024-12-11T16:24:35,615 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/7446a65196a1:0, corePoolSize=5, maxPoolSize=5
2024-12-11T16:24:35,615 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/7446a65196a1:0, corePoolSize=10, maxPoolSize=10
2024-12-11T16:24:35,615 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:24:35,615 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/7446a65196a1:0, corePoolSize=2, maxPoolSize=2
2024-12-11T16:24:35,615 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:24:35,616 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733934305616
2024-12-11T16:24:35,616 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-11T16:24:35,616 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-11T16:24:35,616 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-11T16:24:35,616 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-11T16:24:35,616 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-11T16:24:35,616 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-11T16:24:35,617 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-11T16:24:35,617 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-11T16:24:35,617 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-11T16:24:35,617 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-11T16:24:35,617 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-11T16:24:35,617 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-12-11T16:24:35,617 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-11T16:24:35,617 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-11T16:24:35,618 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/7446a65196a1:0:becomeActiveMaster-HFileCleaner.large.0-1733934275618,5,FailOnTimeoutGroup]
2024-12-11T16:24:35,618 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/7446a65196a1:0:becomeActiveMaster-HFileCleaner.small.0-1733934275618,5,FailOnTimeoutGroup]
2024-12-11T16:24:35,618 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-11T16:24:35,618 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-11T16:24:35,618 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-11T16:24:35,618 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-11T16:24:35,618 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:24:35,618 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-11T16:24:35,626 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37795 is added to blk_1073741831_1007 (size=1321)
2024-12-11T16:24:35,626 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43085 is added to blk_1073741831_1007 (size=1321)
2024-12-11T16:24:35,627 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-12-11T16:24:35,627 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1
2024-12-11T16:24:35,634 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37795 is added to blk_1073741832_1008 (size=32)
2024-12-11T16:24:35,634 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43085 is added to blk_1073741832_1008 (size=32)
2024-12-11T16:24:35,634 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T16:24:35,636 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-11T16:24:35,637 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-11T16:24:35,637 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:24:35,637 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T16:24:35,638 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-11T16:24:35,639 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-11T16:24:35,639 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:24:35,639 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T16:24:35,639 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-11T16:24:35,640 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-11T16:24:35,640 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:24:35,641 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T16:24:35,641 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-11T16:24:35,641 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-11T16:24:35,642 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:24:35,642 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T16:24:35,642 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-11T16:24:35,642 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/data/hbase/meta/1588230740
2024-12-11T16:24:35,643 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/data/hbase/meta/1588230740
2024-12-11T16:24:35,644 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-11T16:24:35,644 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-11T16:24:35,644 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-11T16:24:35,645 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-11T16:24:35,648 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-11T16:24:35,648 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=830349, jitterRate=0.05584344267845154}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-11T16:24:35,648 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1733934275634Initializing all the Stores at 1733934275635 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733934275635Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733934275635Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733934275635Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733934275635Cleaning up temporary data from old regions at 1733934275644 (+9 ms)Region opened successfully at 1733934275648 (+4 ms)
2024-12-11T16:24:35,649 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-11T16:24:35,649 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-11T16:24:35,649 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-11T16:24:35,649 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-11T16:24:35,649 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-11T16:24:35,649 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-11T16:24:35,649 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733934275649Disabling compacts and flushes for region at 1733934275649Disabling writes for close at 1733934275649Writing region close event to WAL at 1733934275649Closed at 1733934275649
2024-12-11T16:24:35,650 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-11T16:24:35,650 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-12-11T16:24:35,650 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-11T16:24:35,651 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-11T16:24:35,652 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-11T16:24:35,678 INFO  [RS:0;7446a65196a1:45491 {}] regionserver.HRegionServer(746): ClusterId : a6d37003-9a9d-410f-8ed6-83d51fe14411
2024-12-11T16:24:35,678 DEBUG [RS:0;7446a65196a1:45491 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-11T16:24:35,710 DEBUG [RS:0;7446a65196a1:45491 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-11T16:24:35,710 DEBUG [RS:0;7446a65196a1:45491 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-11T16:24:35,719 DEBUG [RS:0;7446a65196a1:45491 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-11T16:24:35,719 DEBUG [RS:0;7446a65196a1:45491 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@278930, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7446a65196a1/172.17.0.2:0
2024-12-11T16:24:35,738 DEBUG [RS:0;7446a65196a1:45491 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;7446a65196a1:45491
2024-12-11T16:24:35,738 INFO  [RS:0;7446a65196a1:45491 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-12-11T16:24:35,738 INFO  [RS:0;7446a65196a1:45491 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-12-11T16:24:35,738 DEBUG [RS:0;7446a65196a1:45491 {}] regionserver.HRegionServer(832): About to register with Master.
2024-12-11T16:24:35,739 INFO  [RS:0;7446a65196a1:45491 {}] regionserver.HRegionServer(2659): reportForDuty to master=7446a65196a1,37355,1733934275106 with port=45491, startcode=1733934275250
2024-12-11T16:24:35,739 DEBUG [RS:0;7446a65196a1:45491 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-11T16:24:35,741 INFO  [HMaster-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54787, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.4 (auth:SIMPLE), service=RegionServerStatusService
2024-12-11T16:24:35,741 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37355 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 7446a65196a1,45491,1733934275250
2024-12-11T16:24:35,741 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37355 {}] master.ServerManager(517): Registering regionserver=7446a65196a1,45491,1733934275250
2024-12-11T16:24:35,743 DEBUG [RS:0;7446a65196a1:45491 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1
2024-12-11T16:24:35,743 DEBUG [RS:0;7446a65196a1:45491 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43789
2024-12-11T16:24:35,743 DEBUG [RS:0;7446a65196a1:45491 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-12-11T16:24:35,751 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37355-0x10015bf18ae0000, quorum=127.0.0.1:50451, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-11T16:24:35,751 DEBUG [RS:0;7446a65196a1:45491 {}] zookeeper.ZKUtil(111): regionserver:45491-0x10015bf18ae0001, quorum=127.0.0.1:50451, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/7446a65196a1,45491,1733934275250
2024-12-11T16:24:35,751 WARN  [RS:0;7446a65196a1:45491 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-11T16:24:35,751 INFO  [RS:0;7446a65196a1:45491 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-11T16:24:35,752 DEBUG [RS:0;7446a65196a1:45491 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/WALs/7446a65196a1,45491,1733934275250
2024-12-11T16:24:35,752 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [7446a65196a1,45491,1733934275250]
2024-12-11T16:24:35,756 INFO  [RS:0;7446a65196a1:45491 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-11T16:24:35,758 INFO  [RS:0;7446a65196a1:45491 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-11T16:24:35,758 INFO  [RS:0;7446a65196a1:45491 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-11T16:24:35,758 INFO  [RS:0;7446a65196a1:45491 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-11T16:24:35,758 INFO  [RS:0;7446a65196a1:45491 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-12-11T16:24:35,759 INFO  [RS:0;7446a65196a1:45491 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-12-11T16:24:35,759 INFO  [RS:0;7446a65196a1:45491 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-11T16:24:35,760 DEBUG [RS:0;7446a65196a1:45491 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:24:35,760 DEBUG [RS:0;7446a65196a1:45491 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:24:35,760 DEBUG [RS:0;7446a65196a1:45491 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:24:35,760 DEBUG [RS:0;7446a65196a1:45491 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:24:35,760 DEBUG [RS:0;7446a65196a1:45491 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:24:35,760 DEBUG [RS:0;7446a65196a1:45491 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/7446a65196a1:0, corePoolSize=2, maxPoolSize=2
2024-12-11T16:24:35,760 DEBUG [RS:0;7446a65196a1:45491 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:24:35,760 DEBUG [RS:0;7446a65196a1:45491 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:24:35,760 DEBUG [RS:0;7446a65196a1:45491 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:24:35,760 DEBUG [RS:0;7446a65196a1:45491 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:24:35,760 DEBUG [RS:0;7446a65196a1:45491 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:24:35,761 DEBUG [RS:0;7446a65196a1:45491 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:24:35,761 DEBUG [RS:0;7446a65196a1:45491 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/7446a65196a1:0, corePoolSize=3, maxPoolSize=3
2024-12-11T16:24:35,761 DEBUG [RS:0;7446a65196a1:45491 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/7446a65196a1:0, corePoolSize=3, maxPoolSize=3
2024-12-11T16:24:35,761 INFO  [RS:0;7446a65196a1:45491 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-11T16:24:35,761 INFO  [RS:0;7446a65196a1:45491 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-11T16:24:35,761 INFO  [RS:0;7446a65196a1:45491 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-11T16:24:35,761 INFO  [RS:0;7446a65196a1:45491 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-11T16:24:35,761 INFO  [RS:0;7446a65196a1:45491 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-11T16:24:35,761 INFO  [RS:0;7446a65196a1:45491 {}] hbase.ChoreService(168): Chore ScheduledChore name=7446a65196a1,45491,1733934275250-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-11T16:24:35,777 INFO  [RS:0;7446a65196a1:45491 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-11T16:24:35,777 INFO  [RS:0;7446a65196a1:45491 {}] hbase.ChoreService(168): Chore ScheduledChore name=7446a65196a1,45491,1733934275250-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-11T16:24:35,777 INFO  [RS:0;7446a65196a1:45491 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T16:24:35,777 INFO  [RS:0;7446a65196a1:45491 {}] regionserver.Replication(171): 7446a65196a1,45491,1733934275250 started
2024-12-11T16:24:35,788 INFO  [RS:0;7446a65196a1:45491 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T16:24:35,788 INFO  [RS:0;7446a65196a1:45491 {}] regionserver.HRegionServer(1482): Serving as 7446a65196a1,45491,1733934275250, RpcServer on 7446a65196a1/172.17.0.2:45491, sessionid=0x10015bf18ae0001
2024-12-11T16:24:35,788 DEBUG [RS:0;7446a65196a1:45491 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-11T16:24:35,788 DEBUG [RS:0;7446a65196a1:45491 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 7446a65196a1,45491,1733934275250
2024-12-11T16:24:35,788 DEBUG [RS:0;7446a65196a1:45491 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7446a65196a1,45491,1733934275250'
2024-12-11T16:24:35,788 DEBUG [RS:0;7446a65196a1:45491 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-11T16:24:35,789 DEBUG [RS:0;7446a65196a1:45491 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-11T16:24:35,789 DEBUG [RS:0;7446a65196a1:45491 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-11T16:24:35,789 DEBUG [RS:0;7446a65196a1:45491 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-11T16:24:35,789 DEBUG [RS:0;7446a65196a1:45491 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 7446a65196a1,45491,1733934275250
2024-12-11T16:24:35,789 DEBUG [RS:0;7446a65196a1:45491 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7446a65196a1,45491,1733934275250'
2024-12-11T16:24:35,789 DEBUG [RS:0;7446a65196a1:45491 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-11T16:24:35,790 DEBUG [RS:0;7446a65196a1:45491 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-11T16:24:35,790 DEBUG [RS:0;7446a65196a1:45491 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-11T16:24:35,790 INFO  [RS:0;7446a65196a1:45491 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-11T16:24:35,790 INFO  [RS:0;7446a65196a1:45491 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-11T16:24:35,803 WARN  [7446a65196a1:37355 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-12-11T16:24:35,894 INFO  [RS:0;7446a65196a1:45491 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7446a65196a1%2C45491%2C1733934275250, suffix=, logDir=hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/WALs/7446a65196a1,45491,1733934275250, archiveDir=hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/oldWALs, maxLogs=32
2024-12-11T16:24:35,896 INFO  [RS:0;7446a65196a1:45491 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7446a65196a1%2C45491%2C1733934275250.1733934275895
2024-12-11T16:24:36,013 INFO  [RS:0;7446a65196a1:45491 {}] wal.AbstractFSWAL(1368): Slow sync cost: 114 ms, current pipeline: []
2024-12-11T16:24:36,013 INFO  [RS:0;7446a65196a1:45491 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/WALs/7446a65196a1,45491,1733934275250/7446a65196a1%2C45491%2C1733934275250.1733934275895
2024-12-11T16:24:36,014 DEBUG [RS:0;7446a65196a1:45491 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35247:35247),(127.0.0.1/127.0.0.1:34645:34645)]
2024-12-11T16:24:36,053 DEBUG [7446a65196a1:37355 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-12-11T16:24:36,053 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=7446a65196a1,45491,1733934275250
2024-12-11T16:24:36,054 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7446a65196a1,45491,1733934275250, state=OPENING
2024-12-11T16:24:36,306 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:24:36,319 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:24:36,364 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-11T16:24:36,419 DEBUG [pool-525-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45491-0x10015bf18ae0001, quorum=127.0.0.1:50451, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:24:36,419 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37355-0x10015bf18ae0000, quorum=127.0.0.1:50451, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:24:36,420 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-11T16:24:36,420 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-11T16:24:36,420 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-11T16:24:36,420 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=7446a65196a1,45491,1733934275250}]
2024-12-11T16:24:36,582 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-11T16:24:36,591 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46755, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-11T16:24:36,596 INFO  [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-12-11T16:24:36,596 INFO  [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-11T16:24:36,598 INFO  [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7446a65196a1%2C45491%2C1733934275250.meta, suffix=.meta, logDir=hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/WALs/7446a65196a1,45491,1733934275250, archiveDir=hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/oldWALs, maxLogs=32
2024-12-11T16:24:36,599 INFO  [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 7446a65196a1%2C45491%2C1733934275250.meta.1733934276599.meta
2024-12-11T16:24:36,610 INFO  [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/WALs/7446a65196a1,45491,1733934275250/7446a65196a1%2C45491%2C1733934275250.meta.1733934276599.meta
2024-12-11T16:24:36,613 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34645:34645),(127.0.0.1/127.0.0.1:35247:35247)]
2024-12-11T16:24:36,614 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-11T16:24:36,614 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-11T16:24:36,614 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-11T16:24:36,615 INFO  [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-11T16:24:36,615 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-11T16:24:36,615 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T16:24:36,615 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-12-11T16:24:36,615 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-12-11T16:24:36,617 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-11T16:24:36,618 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-11T16:24:36,618 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:24:36,619 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T16:24:36,619 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-11T16:24:36,620 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-11T16:24:36,620 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:24:36,621 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T16:24:36,621 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-11T16:24:36,622 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-11T16:24:36,622 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:24:36,623 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T16:24:36,623 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-11T16:24:36,624 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-11T16:24:36,624 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:24:36,625 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T16:24:36,625 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-11T16:24:36,626 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/data/hbase/meta/1588230740
2024-12-11T16:24:36,627 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/data/hbase/meta/1588230740
2024-12-11T16:24:36,628 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-11T16:24:36,629 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-11T16:24:36,629 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-11T16:24:36,631 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-11T16:24:36,632 INFO  [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=714212, jitterRate=-0.09183259308338165}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-11T16:24:36,632 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-12-11T16:24:36,633 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1733934276615Writing region info on filesystem at 1733934276615Initializing all the Stores at 1733934276616 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733934276616Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733934276617 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733934276617Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733934276617Cleaning up temporary data from old regions at 1733934276629 (+12 ms)Running coprocessor post-open hooks at 1733934276632 (+3 ms)Region opened successfully at 1733934276633 (+1 ms)
2024-12-11T16:24:36,634 INFO  [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733934276581
2024-12-11T16:24:36,636 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-11T16:24:36,637 INFO  [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-12-11T16:24:36,638 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=7446a65196a1,45491,1733934275250
2024-12-11T16:24:36,639 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7446a65196a1,45491,1733934275250, state=OPEN
2024-12-11T16:24:36,690 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37355-0x10015bf18ae0000, quorum=127.0.0.1:50451, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-11T16:24:36,690 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-11T16:24:36,690 DEBUG [pool-525-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45491-0x10015bf18ae0001, quorum=127.0.0.1:50451, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-11T16:24:36,690 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=7446a65196a1,45491,1733934275250
2024-12-11T16:24:36,691 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-11T16:24:36,697 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-12-11T16:24:36,697 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7446a65196a1,45491,1733934275250 in 270 msec
2024-12-11T16:24:36,701 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-12-11T16:24:36,701 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 1.0470 sec
2024-12-11T16:24:36,702 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-11T16:24:36,702 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-12-11T16:24:36,704 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-11T16:24:36,704 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7446a65196a1,45491,1733934275250, seqNum=-1]
2024-12-11T16:24:36,704 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-11T16:24:36,706 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-11-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37691, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-11T16:24:36,711 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0970 sec
2024-12-11T16:24:36,711 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733934276711, completionTime=-1
2024-12-11T16:24:36,712 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-12-11T16:24:36,712 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-12-11T16:24:36,714 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-12-11T16:24:36,714 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733934336714
2024-12-11T16:24:36,714 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733934396714
2024-12-11T16:24:36,714 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec
2024-12-11T16:24:36,715 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7446a65196a1,37355,1733934275106-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-11T16:24:36,715 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7446a65196a1,37355,1733934275106-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T16:24:36,715 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7446a65196a1,37355,1733934275106-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T16:24:36,715 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-7446a65196a1:37355, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T16:24:36,715 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-11T16:24:36,717 DEBUG [master/7446a65196a1:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-12-11T16:24:36,720 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-12-11T16:24:36,724 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.415sec
2024-12-11T16:24:36,725 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-11T16:24:36,725 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-11T16:24:36,725 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-11T16:24:36,725 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-11T16:24:36,725 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-11T16:24:36,725 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7446a65196a1,37355,1733934275106-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-11T16:24:36,725 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7446a65196a1,37355,1733934275106-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-11T16:24:36,728 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-12-11T16:24:36,728 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-11T16:24:36,728 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7446a65196a1,37355,1733934275106-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T16:24:36,778 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59ca75fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-11T16:24:36,779 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7446a65196a1,37355,-1 for getting cluster id
2024-12-11T16:24:36,779 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-12-11T16:24:36,785 DEBUG [HMaster-EventLoopGroup-10-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a6d37003-9a9d-410f-8ed6-83d51fe14411'
2024-12-11T16:24:36,785 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-12-11T16:24:36,786 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a6d37003-9a9d-410f-8ed6-83d51fe14411"

2024-12-11T16:24:36,786 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67bdfa8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-11T16:24:36,786 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7446a65196a1,37355,-1]
2024-12-11T16:24:36,786 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-12-11T16:24:36,787 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T16:24:36,789 INFO  [HMaster-EventLoopGroup-10-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52828, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-12-11T16:24:36,790 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71c339ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-11T16:24:36,791 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-11T16:24:36,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7446a65196a1,45491,1733934275250, seqNum=-1]
2024-12-11T16:24:36,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-11T16:24:36,795 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55598, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-11T16:24:36,797 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=7446a65196a1,37355,1733934275106
2024-12-11T16:24:36,798 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T16:24:36,801 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-12-11T16:24:36,801 INFO  [Time-limited test {}] wal.TestLogRolling(320): Starting testLogRollOnPipelineRestart
2024-12-11T16:24:36,801 INFO  [Time-limited test {}] wal.TestLogRolling(323): Replication=2
2024-12-11T16:24:36,801 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-12-11T16:24:36,803 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncConnectionImpl(321): The fetched master address is 7446a65196a1,37355,1733934275106
2024-12-11T16:24:36,803 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@69e5fa7f
2024-12-11T16:24:36,803 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-12-11T16:24:36,809 INFO  [HMaster-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52836, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-12-11T16:24:36,809 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-12-11T16:24:36,809 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-12-11T16:24:36,809 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRollOnPipelineRestart', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-11T16:24:36,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart
2024-12-11T16:24:36,813 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_PRE_OPERATION
2024-12-11T16:24:36,813 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:24:36,813 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRollOnPipelineRestart"
 procId is: 4
2024-12-11T16:24:36,814 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-12-11T16:24:36,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-11T16:24:36,826 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37795 is added to blk_1073741835_1011 (size=395)
2024-12-11T16:24:36,828 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43085 is added to blk_1073741835_1011 (size=395)
2024-12-11T16:24:36,831 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 455cd7920d80c3fa69ca86699047709c, NAME => 'TestLogRolling-testLogRollOnPipelineRestart,,1733934276809.455cd7920d80c3fa69ca86699047709c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnPipelineRestart', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1
2024-12-11T16:24:36,848 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43085 is added to blk_1073741836_1012 (size=78)
2024-12-11T16:24:36,848 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37795 is added to blk_1073741836_1012 (size=78)
2024-12-11T16:24:36,850 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnPipelineRestart,,1733934276809.455cd7920d80c3fa69ca86699047709c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T16:24:36,850 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1722): Closing 455cd7920d80c3fa69ca86699047709c, disabling compactions & flushes
2024-12-11T16:24:36,850 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1733934276809.455cd7920d80c3fa69ca86699047709c.
2024-12-11T16:24:36,850 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1733934276809.455cd7920d80c3fa69ca86699047709c.
2024-12-11T16:24:36,850 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1733934276809.455cd7920d80c3fa69ca86699047709c. after waiting 0 ms
2024-12-11T16:24:36,850 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1733934276809.455cd7920d80c3fa69ca86699047709c.
2024-12-11T16:24:36,850 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnPipelineRestart,,1733934276809.455cd7920d80c3fa69ca86699047709c.
2024-12-11T16:24:36,850 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1676): Region close journal for 455cd7920d80c3fa69ca86699047709c:
Waiting for close lock at 1733934276850Disabling compacts and flushes for region at 1733934276850Disabling writes for close at 1733934276850Writing region close event to WAL at 1733934276850Closed at 1733934276850
2024-12-11T16:24:36,851 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_ADD_TO_META
2024-12-11T16:24:36,852 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnPipelineRestart,,1733934276809.455cd7920d80c3fa69ca86699047709c.","families":{"info":[{"qualifier":"regioninfo","vlen":77,"tag":[],"timestamp":"1733934276851"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733934276851"}]},"ts":"1733934276851"}
2024-12-11T16:24:36,855 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-12-11T16:24:36,856 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-12-11T16:24:36,857 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnPipelineRestart","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733934276856"}]},"ts":"1733934276856"}
2024-12-11T16:24:36,860 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnPipelineRestart, state=ENABLING in hbase:meta
2024-12-11T16:24:36,860 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=455cd7920d80c3fa69ca86699047709c, ASSIGN}]
2024-12-11T16:24:36,862 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=455cd7920d80c3fa69ca86699047709c, ASSIGN
2024-12-11T16:24:36,863 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=455cd7920d80c3fa69ca86699047709c, ASSIGN; state=OFFLINE, location=7446a65196a1,45491,1733934275250; forceNewPlan=false, retain=false
2024-12-11T16:24:37,014 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=455cd7920d80c3fa69ca86699047709c, regionState=OPENING, regionLocation=7446a65196a1,45491,1733934275250
2024-12-11T16:24:37,018 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-11-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=455cd7920d80c3fa69ca86699047709c, ASSIGN because future has completed
2024-12-11T16:24:37,019 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 455cd7920d80c3fa69ca86699047709c, server=7446a65196a1,45491,1733934275250}]
2024-12-11T16:24:37,177 INFO  [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRollOnPipelineRestart,,1733934276809.455cd7920d80c3fa69ca86699047709c.
2024-12-11T16:24:37,177 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 455cd7920d80c3fa69ca86699047709c, NAME => 'TestLogRolling-testLogRollOnPipelineRestart,,1733934276809.455cd7920d80c3fa69ca86699047709c.', STARTKEY => '', ENDKEY => ''}
2024-12-11T16:24:37,177 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnPipelineRestart 455cd7920d80c3fa69ca86699047709c
2024-12-11T16:24:37,177 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnPipelineRestart,,1733934276809.455cd7920d80c3fa69ca86699047709c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T16:24:37,177 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 455cd7920d80c3fa69ca86699047709c
2024-12-11T16:24:37,177 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 455cd7920d80c3fa69ca86699047709c
2024-12-11T16:24:37,179 INFO  [StoreOpener-455cd7920d80c3fa69ca86699047709c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 455cd7920d80c3fa69ca86699047709c 
2024-12-11T16:24:37,181 INFO  [StoreOpener-455cd7920d80c3fa69ca86699047709c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 455cd7920d80c3fa69ca86699047709c columnFamilyName info
2024-12-11T16:24:37,181 DEBUG [StoreOpener-455cd7920d80c3fa69ca86699047709c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:24:37,182 INFO  [StoreOpener-455cd7920d80c3fa69ca86699047709c-1 {}] regionserver.HStore(327): Store=455cd7920d80c3fa69ca86699047709c/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-11T16:24:37,182 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 455cd7920d80c3fa69ca86699047709c
2024-12-11T16:24:37,183 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/data/default/TestLogRolling-testLogRollOnPipelineRestart/455cd7920d80c3fa69ca86699047709c
2024-12-11T16:24:37,184 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/data/default/TestLogRolling-testLogRollOnPipelineRestart/455cd7920d80c3fa69ca86699047709c
2024-12-11T16:24:37,184 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 455cd7920d80c3fa69ca86699047709c
2024-12-11T16:24:37,184 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 455cd7920d80c3fa69ca86699047709c
2024-12-11T16:24:37,186 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 455cd7920d80c3fa69ca86699047709c
2024-12-11T16:24:37,189 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/data/default/TestLogRolling-testLogRollOnPipelineRestart/455cd7920d80c3fa69ca86699047709c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-11T16:24:37,189 INFO  [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 455cd7920d80c3fa69ca86699047709c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=847409, jitterRate=0.07753722369670868}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-11T16:24:37,189 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 455cd7920d80c3fa69ca86699047709c
2024-12-11T16:24:37,190 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 455cd7920d80c3fa69ca86699047709c:
Running coprocessor pre-open hook at 1733934277178Writing region info on filesystem at 1733934277178Initializing all the Stores at 1733934277178Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733934277179 (+1 ms)Cleaning up temporary data from old regions at 1733934277184 (+5 ms)Running coprocessor post-open hooks at 1733934277189 (+5 ms)Region opened successfully at 1733934277190 (+1 ms)
2024-12-11T16:24:37,191 INFO  [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRollOnPipelineRestart,,1733934276809.455cd7920d80c3fa69ca86699047709c., pid=6, masterSystemTime=1733934277172
2024-12-11T16:24:37,191 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart
2024-12-11T16:24:37,192 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart Metrics about Tables on a single HBase RegionServer
2024-12-11T16:24:37,192 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath
2024-12-11T16:24:37,194 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRollOnPipelineRestart,,1733934276809.455cd7920d80c3fa69ca86699047709c.
2024-12-11T16:24:37,194 INFO  [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRollOnPipelineRestart,,1733934276809.455cd7920d80c3fa69ca86699047709c.
2024-12-11T16:24:37,195 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=455cd7920d80c3fa69ca86699047709c, regionState=OPEN, openSeqNum=2, regionLocation=7446a65196a1,45491,1733934275250
2024-12-11T16:24:37,197 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-11-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 455cd7920d80c3fa69ca86699047709c, server=7446a65196a1,45491,1733934275250 because future has completed
2024-12-11T16:24:37,201 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-12-11T16:24:37,201 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 455cd7920d80c3fa69ca86699047709c, server=7446a65196a1,45491,1733934275250 in 180 msec
2024-12-11T16:24:37,204 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-12-11T16:24:37,204 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=455cd7920d80c3fa69ca86699047709c, ASSIGN in 341 msec
2024-12-11T16:24:37,205 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-12-11T16:24:37,205 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnPipelineRestart","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733934277205"}]},"ts":"1733934277205"}
2024-12-11T16:24:37,207 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnPipelineRestart, state=ENABLED in hbase:meta
2024-12-11T16:24:37,209 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_POST_OPERATION
2024-12-11T16:24:37,211 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart in 400 msec
2024-12-11T16:24:37,307 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:24:37,319 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:24:38,308 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:24:38,320 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:24:39,308 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:24:39,321 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:24:40,309 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:24:40,322 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:24:41,310 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:24:41,323 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:24:42,117 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-11T16:24:42,142 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:24:42,143 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:24:42,143 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:24:42,143 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:24:42,144 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:24:42,144 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:24:42,148 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:24:42,148 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:24:42,148 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:24:42,153 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:24:42,161 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-12-11T16:24:42,161 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnPipelineRestart'
2024-12-11T16:24:42,310 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:24:42,323 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:24:43,311 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:24:43,324 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:24:44,312 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:24:44,325 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:24:45,312 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:24:45,326 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:24:46,313 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:24:46,327 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:24:46,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-11T16:24:46,841 INFO  [RPCClient-NioEventLoopGroup-4-15 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnPipelineRestart completed
2024-12-11T16:24:46,841 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRollOnPipelineRestart,, stopping at row=TestLogRolling-testLogRollOnPipelineRestart ,, for max=2147483647 with caching=100
2024-12-11T16:24:46,845 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRollOnPipelineRestart
2024-12-11T16:24:46,845 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRollOnPipelineRestart,,1733934276809.455cd7920d80c3fa69ca86699047709c.
2024-12-11T16:24:46,850 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRollOnPipelineRestart', row='row1002', locateType=CURRENT is [region=TestLogRolling-testLogRollOnPipelineRestart,,1733934276809.455cd7920d80c3fa69ca86699047709c., hostname=7446a65196a1,45491,1733934275250, seqNum=2]
2024-12-11T16:24:47,314 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:24:47,327 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:24:48,315 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:24:48,328 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:24:48,852 INFO  [Time-limited test {}] wal.TestLogRolling(360): log.getCurrentFileName()): hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/WALs/7446a65196a1,45491,1733934275250/7446a65196a1%2C45491%2C1733934275250.1733934275895
2024-12-11T16:24:48,853 WARN  [ResponseProcessor for block BP-119074730-172.17.0.2-1733934273371:blk_1073741830_1006 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-119074730-172.17.0.2-1733934273371:blk_1073741830_1006
java.io.IOException: Bad response ERROR for BP-119074730-172.17.0.2-1733934273371:blk_1073741830_1006 from datanode DatanodeInfoWithStorage[127.0.0.1:37795,DS-31d80686-d824-4918-894d-bf029f453b89,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:48,853 WARN  [ResponseProcessor for block BP-119074730-172.17.0.2-1733934273371:blk_1073741833_1009 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-119074730-172.17.0.2-1733934273371:blk_1073741833_1009
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:48,853 WARN  [DataStreamer for file /user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/WALs/7446a65196a1,45491,1733934275250/7446a65196a1%2C45491%2C1733934275250.1733934275895 block BP-119074730-172.17.0.2-1733934273371:blk_1073741833_1009 {}] hdfs.DataStreamer(1731): Error Recovery for BP-119074730-172.17.0.2-1733934273371:blk_1073741833_1009 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37795,DS-31d80686-d824-4918-894d-bf029f453b89,DISK], DatanodeInfoWithStorage[127.0.0.1:43085,DS-d0922336-6ec8-41bd-9084-f220c6cd95f5,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37795,DS-31d80686-d824-4918-894d-bf029f453b89,DISK]) is bad.
2024-12-11T16:24:48,853 WARN  [ResponseProcessor for block BP-119074730-172.17.0.2-1733934273371:blk_1073741834_1010 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-119074730-172.17.0.2-1733934273371:blk_1073741834_1010
java.io.IOException: Bad response ERROR for BP-119074730-172.17.0.2-1733934273371:blk_1073741834_1010 from datanode DatanodeInfoWithStorage[127.0.0.1:37795,DS-31d80686-d824-4918-894d-bf029f453b89,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:48,854 WARN  [DataStreamer for file /user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/MasterData/WALs/7446a65196a1,37355,1733934275106/7446a65196a1%2C37355%2C1733934275106.1733934275402 block BP-119074730-172.17.0.2-1733934273371:blk_1073741830_1006 {}] hdfs.DataStreamer(1731): Error Recovery for BP-119074730-172.17.0.2-1733934273371:blk_1073741830_1006 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43085,DS-d0922336-6ec8-41bd-9084-f220c6cd95f5,DISK], DatanodeInfoWithStorage[127.0.0.1:37795,DS-31d80686-d824-4918-894d-bf029f453b89,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:37795,DS-31d80686-d824-4918-894d-bf029f453b89,DISK]) is bad.
2024-12-11T16:24:48,854 WARN  [DataStreamer for file /user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/WALs/7446a65196a1,45491,1733934275250/7446a65196a1%2C45491%2C1733934275250.meta.1733934276599.meta block BP-119074730-172.17.0.2-1733934273371:blk_1073741834_1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-119074730-172.17.0.2-1733934273371:blk_1073741834_1010 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43085,DS-d0922336-6ec8-41bd-9084-f220c6cd95f5,DISK], DatanodeInfoWithStorage[127.0.0.1:37795,DS-31d80686-d824-4918-894d-bf029f453b89,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:37795,DS-31d80686-d824-4918-894d-bf029f453b89,DISK]) is bad.
2024-12-11T16:24:48,854 WARN  [PacketResponder: BP-119074730-172.17.0.2-1733934273371:blk_1073741830_1006, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:37795] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T16:24:48,855 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1292506435_22 at /127.0.0.1:43028 [Receiving block BP-119074730-172.17.0.2-1733934273371:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:43085:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:43028 dst: /127.0.0.1:43085
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T16:24:48,855 WARN  [PacketResponder: BP-119074730-172.17.0.2-1733934273371:blk_1073741834_1010, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:37795] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T16:24:48,855 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-928974549_22 at /127.0.0.1:43048 [Receiving block BP-119074730-172.17.0.2-1733934273371:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:43085:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:43048 dst: /127.0.0.1:43085
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T16:24:48,855 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1292506435_22 at /127.0.0.1:51602 [Receiving block BP-119074730-172.17.0.2-1733934273371:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:37795:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:51602 dst: /127.0.0.1:37795
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T16:24:48,855 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-928974549_22 at /127.0.0.1:51624 [Receiving block BP-119074730-172.17.0.2-1733934273371:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:37795:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:51624 dst: /127.0.0.1:37795
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T16:24:48,856 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-928974549_22 at /127.0.0.1:43046 [Receiving block BP-119074730-172.17.0.2-1733934273371:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:43085:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:43046 dst: /127.0.0.1:43085
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T16:24:48,855 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-928974549_22 at /127.0.0.1:51628 [Receiving block BP-119074730-172.17.0.2-1733934273371:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:37795:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:51628 dst: /127.0.0.1:37795
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T16:24:48,858 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@30a95171{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T16:24:48,861 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4586c70{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-11T16:24:48,861 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-11T16:24:48,861 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@61ecc897{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-11T16:24:48,861 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3a6fb45d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90c660c-600a-9166-c37d-1b9c62c53b37/hadoop.log.dir/,STOPPED}
2024-12-11T16:24:48,862 WARN  [BP-119074730-172.17.0.2-1733934273371 heartbeating to localhost/127.0.0.1:43789 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-11T16:24:48,862 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-11T16:24:48,862 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-11T16:24:48,862 WARN  [BP-119074730-172.17.0.2-1733934273371 heartbeating to localhost/127.0.0.1:43789 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-119074730-172.17.0.2-1733934273371 (Datanode Uuid 518c53e3-a44d-421f-a0df-e28575189a25) service to localhost/127.0.0.1:43789
2024-12-11T16:24:48,863 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90c660c-600a-9166-c37d-1b9c62c53b37/cluster_c6aa3bec-caf4-e30f-1389-b14fd52f58cf/data/data3/current/BP-119074730-172.17.0.2-1733934273371 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T16:24:48,863 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90c660c-600a-9166-c37d-1b9c62c53b37/cluster_c6aa3bec-caf4-e30f-1389-b14fd52f58cf/data/data4/current/BP-119074730-172.17.0.2-1733934273371 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T16:24:48,863 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-11T16:24:48,873 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-11T16:24:48,877 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-11T16:24:48,878 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-11T16:24:48,878 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-11T16:24:48,878 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-11T16:24:48,880 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@17d86da6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90c660c-600a-9166-c37d-1b9c62c53b37/hadoop.log.dir/,AVAILABLE}
2024-12-11T16:24:48,880 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6e094446{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-11T16:24:48,997 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@54dc8c82{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90c660c-600a-9166-c37d-1b9c62c53b37/java.io.tmpdir/jetty-localhost-43815-hadoop-hdfs-3_4_1-tests_jar-_-any-11402586099424964043/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T16:24:48,998 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1d5f8483{HTTP/1.1, (http/1.1)}{localhost:43815}
2024-12-11T16:24:48,998 INFO  [Time-limited test {}] server.Server(415): Started @179668ms
2024-12-11T16:24:48,999 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-11T16:24:49,047 WARN  [ResponseProcessor for block BP-119074730-172.17.0.2-1733934273371:blk_1073741833_1014 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-119074730-172.17.0.2-1733934273371:blk_1073741833_1014
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:49,047 WARN  [ResponseProcessor for block BP-119074730-172.17.0.2-1733934273371:blk_1073741834_1013 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-119074730-172.17.0.2-1733934273371:blk_1073741834_1013
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:49,047 WARN  [ResponseProcessor for block BP-119074730-172.17.0.2-1733934273371:blk_1073741830_1015 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-119074730-172.17.0.2-1733934273371:blk_1073741830_1015
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:49,048 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-928974549_22 at /127.0.0.1:39312 [Receiving block BP-119074730-172.17.0.2-1733934273371:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:43085:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:39312 dst: /127.0.0.1:43085
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T16:24:49,048 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1292506435_22 at /127.0.0.1:39320 [Receiving block BP-119074730-172.17.0.2-1733934273371:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:43085:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:39320 dst: /127.0.0.1:43085
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T16:24:49,048 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-928974549_22 at /127.0.0.1:39310 [Receiving block BP-119074730-172.17.0.2-1733934273371:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:43085:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:39310 dst: /127.0.0.1:43085
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T16:24:49,056 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@124f9706{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T16:24:49,056 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@52bf6dde{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-11T16:24:49,056 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-11T16:24:49,056 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5a617dcf{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-11T16:24:49,057 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@520d4f99{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90c660c-600a-9166-c37d-1b9c62c53b37/hadoop.log.dir/,STOPPED}
2024-12-11T16:24:49,060 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-11T16:24:49,060 WARN  [BP-119074730-172.17.0.2-1733934273371 heartbeating to localhost/127.0.0.1:43789 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-11T16:24:49,060 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-11T16:24:49,060 WARN  [BP-119074730-172.17.0.2-1733934273371 heartbeating to localhost/127.0.0.1:43789 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-119074730-172.17.0.2-1733934273371 (Datanode Uuid 94f6fc17-0a75-416c-8864-c6b69c7ce840) service to localhost/127.0.0.1:43789
2024-12-11T16:24:49,060 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90c660c-600a-9166-c37d-1b9c62c53b37/cluster_c6aa3bec-caf4-e30f-1389-b14fd52f58cf/data/data1/current/BP-119074730-172.17.0.2-1733934273371 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T16:24:49,061 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90c660c-600a-9166-c37d-1b9c62c53b37/cluster_c6aa3bec-caf4-e30f-1389-b14fd52f58cf/data/data2/current/BP-119074730-172.17.0.2-1733934273371 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T16:24:49,061 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-11T16:24:49,074 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-11T16:24:49,077 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-11T16:24:49,081 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-11T16:24:49,081 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-11T16:24:49,081 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-11T16:24:49,082 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@44d8c74f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90c660c-600a-9166-c37d-1b9c62c53b37/hadoop.log.dir/,AVAILABLE}
2024-12-11T16:24:49,083 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@75c3577f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-11T16:24:49,192 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1b350de2{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90c660c-600a-9166-c37d-1b9c62c53b37/java.io.tmpdir/jetty-localhost-37789-hadoop-hdfs-3_4_1-tests_jar-_-any-10089427870462409901/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T16:24:49,193 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7cfbc071{HTTP/1.1, (http/1.1)}{localhost:37789}
2024-12-11T16:24:49,193 INFO  [Time-limited test {}] server.Server(415): Started @179863ms
2024-12-11T16:24:49,194 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-11T16:24:49,316 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:24:49,329 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:24:49,431 WARN  [Thread-1343 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-11T16:24:49,434 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5240e4b27955c7bf with lease ID 0x5a0312432928716d: from storage DS-31d80686-d824-4918-894d-bf029f453b89 node DatanodeRegistration(127.0.0.1:37399, datanodeUuid=518c53e3-a44d-421f-a0df-e28575189a25, infoPort=42819, infoSecurePort=0, ipcPort=37817, storageInfo=lv=-57;cid=testClusterID;nsid=6436803;c=1733934273371), blocks: 6, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-12-11T16:24:49,434 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5240e4b27955c7bf with lease ID 0x5a0312432928716d: from storage DS-008cd667-4edc-4c90-92f3-4613f09bac93 node DatanodeRegistration(127.0.0.1:37399, datanodeUuid=518c53e3-a44d-421f-a0df-e28575189a25, infoPort=42819, infoSecurePort=0, ipcPort=37817, storageInfo=lv=-57;cid=testClusterID;nsid=6436803;c=1733934273371), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-11T16:24:49,745 WARN  [Thread-1364 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-11T16:24:49,749 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2edb3a8143baf058 with lease ID 0x5a0312432928716e: from storage DS-d0922336-6ec8-41bd-9084-f220c6cd95f5 node DatanodeRegistration(127.0.0.1:33753, datanodeUuid=94f6fc17-0a75-416c-8864-c6b69c7ce840, infoPort=38503, infoSecurePort=0, ipcPort=34131, storageInfo=lv=-57;cid=testClusterID;nsid=6436803;c=1733934273371), blocks: 6, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-12-11T16:24:49,749 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2edb3a8143baf058 with lease ID 0x5a0312432928716e: from storage DS-eb41716c-cf35-4f8a-a3bd-166a79ec6316 node DatanodeRegistration(127.0.0.1:33753, datanodeUuid=94f6fc17-0a75-416c-8864-c6b69c7ce840, infoPort=38503, infoSecurePort=0, ipcPort=34131, storageInfo=lv=-57;cid=testClusterID;nsid=6436803;c=1733934273371), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-11T16:24:50,228 INFO  [Time-limited test {}] wal.TestLogRolling(372): Data Nodes restarted
2024-12-11T16:24:50,232 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1002
2024-12-11T16:24:50,234 ERROR [FSHLog-0-hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1-prefix:7446a65196a1,45491,1733934275250 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43085,DS-d0922336-6ec8-41bd-9084-f220c6cd95f5,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:50,234 WARN  [FSHLog-0-hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1-prefix:7446a65196a1,45491,1733934275250 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43085,DS-d0922336-6ec8-41bd-9084-f220c6cd95f5,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:50,235 DEBUG [regionserver/7446a65196a1:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 7446a65196a1%2C45491%2C1733934275250:(num 1733934275895) roll requested
2024-12-11T16:24:50,235 INFO  [regionserver/7446a65196a1:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7446a65196a1%2C45491%2C1733934275250.1733934290235
2024-12-11T16:24:50,260 DEBUG [regionserver/7446a65196a1:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/WALs/7446a65196a1,45491,1733934275250/7446a65196a1%2C45491%2C1733934275250.1733934275895 newFile=hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/WALs/7446a65196a1,45491,1733934275250/7446a65196a1%2C45491%2C1733934275250.1733934290235
2024-12-11T16:24:50,260 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:24:50,261 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:24:50,261 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:24:50,261 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:24:50,261 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:24:50,261 INFO  [regionserver/7446a65196a1:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/WALs/7446a65196a1,45491,1733934275250/7446a65196a1%2C45491%2C1733934275250.1733934275895 with entries=2, filesize=1.59 KB; new WAL /user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/WALs/7446a65196a1,45491,1733934275250/7446a65196a1%2C45491%2C1733934275250.1733934290235
2024-12-11T16:24:50,261 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43085,DS-d0922336-6ec8-41bd-9084-f220c6cd95f5,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:50,262 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43085,DS-d0922336-6ec8-41bd-9084-f220c6cd95f5,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:50,262 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/WALs/7446a65196a1,45491,1733934275250/7446a65196a1%2C45491%2C1733934275250.1733934275895
2024-12-11T16:24:50,262 WARN  [IPC Server handler 0 on default port 43789 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/WALs/7446a65196a1,45491,1733934275250/7446a65196a1%2C45491%2C1733934275250.1733934275895 has not been closed. Lease recovery is in progress. RecoveryId = 1017 for block blk_1073741833_1014
2024-12-11T16:24:50,263 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/WALs/7446a65196a1,45491,1733934275250/7446a65196a1%2C45491%2C1733934275250.1733934275895 after 1ms
2024-12-11T16:24:50,264 DEBUG [regionserver/7446a65196a1:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38503:38503),(127.0.0.1/127.0.0.1:42819:42819)]
2024-12-11T16:24:50,264 DEBUG [regionserver/7446a65196a1:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/WALs/7446a65196a1,45491,1733934275250/7446a65196a1%2C45491%2C1733934275250.1733934275895 is not closed yet, will try archiving it next time
2024-12-11T16:24:50,316 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:24:50,330 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:24:51,317 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:24:51,331 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:24:52,268 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1003
2024-12-11T16:24:52,318 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:24:52,332 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:24:52,434 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741833_1014: GenerationStamp not matched, existing replica is blk_1073741833_1009
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-12-11T16:24:53,318 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:24:53,332 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:24:54,264 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/WALs/7446a65196a1,45491,1733934275250/7446a65196a1%2C45491%2C1733934275250.1733934275895 after 4002ms
2024-12-11T16:24:54,271 WARN  [ResponseProcessor for block BP-119074730-172.17.0.2-1733934273371:blk_1073741837_1016 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-119074730-172.17.0.2-1733934273371:blk_1073741837_1016
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:54,271 WARN  [DataStreamer for file /user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/WALs/7446a65196a1,45491,1733934275250/7446a65196a1%2C45491%2C1733934275250.1733934290235 block BP-119074730-172.17.0.2-1733934273371:blk_1073741837_1016 {}] hdfs.DataStreamer(1731): Error Recovery for BP-119074730-172.17.0.2-1733934273371:blk_1073741837_1016 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33753,DS-d0922336-6ec8-41bd-9084-f220c6cd95f5,DISK], DatanodeInfoWithStorage[127.0.0.1:37399,DS-31d80686-d824-4918-894d-bf029f453b89,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33753,DS-d0922336-6ec8-41bd-9084-f220c6cd95f5,DISK]) is bad.
2024-12-11T16:24:54,272 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-928974549_22 at /127.0.0.1:58048 [Receiving block BP-119074730-172.17.0.2-1733934273371:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:33753:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:58048 dst: /127.0.0.1:33753
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T16:24:54,272 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-928974549_22 at /127.0.0.1:46200 [Receiving block BP-119074730-172.17.0.2-1733934273371:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:37399:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:46200 dst: /127.0.0.1:37399
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T16:24:54,309 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1b350de2{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T16:24:54,309 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7cfbc071{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-11T16:24:54,309 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-11T16:24:54,310 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@75c3577f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-11T16:24:54,311 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@44d8c74f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90c660c-600a-9166-c37d-1b9c62c53b37/hadoop.log.dir/,STOPPED}
2024-12-11T16:24:54,322 WARN  [BP-119074730-172.17.0.2-1733934273371 heartbeating to localhost/127.0.0.1:43789 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-11T16:24:54,322 WARN  [BP-119074730-172.17.0.2-1733934273371 heartbeating to localhost/127.0.0.1:43789 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-119074730-172.17.0.2-1733934273371 (Datanode Uuid 94f6fc17-0a75-416c-8864-c6b69c7ce840) service to localhost/127.0.0.1:43789
2024-12-11T16:24:54,322 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:24:54,324 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90c660c-600a-9166-c37d-1b9c62c53b37/cluster_c6aa3bec-caf4-e30f-1389-b14fd52f58cf/data/data1/current/BP-119074730-172.17.0.2-1733934273371 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T16:24:54,324 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90c660c-600a-9166-c37d-1b9c62c53b37/cluster_c6aa3bec-caf4-e30f-1389-b14fd52f58cf/data/data2/current/BP-119074730-172.17.0.2-1733934273371 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T16:24:54,324 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-11T16:24:54,327 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-11T16:24:54,327 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-11T16:24:54,333 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:24:54,349 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-11T16:24:54,356 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-11T16:24:54,359 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-11T16:24:54,359 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-11T16:24:54,359 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-11T16:24:54,360 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2933b309{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90c660c-600a-9166-c37d-1b9c62c53b37/hadoop.log.dir/,AVAILABLE}
2024-12-11T16:24:54,360 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@437e5421{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-11T16:24:54,558 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@581007a9{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90c660c-600a-9166-c37d-1b9c62c53b37/java.io.tmpdir/jetty-localhost-44273-hadoop-hdfs-3_4_1-tests_jar-_-any-9242398208090867874/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T16:24:54,558 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@f40aa31{HTTP/1.1, (http/1.1)}{localhost:44273}
2024-12-11T16:24:54,559 INFO  [Time-limited test {}] server.Server(415): Started @185229ms
2024-12-11T16:24:54,560 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-11T16:24:54,647 WARN  [ResponseProcessor for block BP-119074730-172.17.0.2-1733934273371:blk_1073741837_1018 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-119074730-172.17.0.2-1733934273371:blk_1073741837_1018
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:54,652 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-928974549_22 at /127.0.0.1:51908 [Receiving block BP-119074730-172.17.0.2-1733934273371:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:37399:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:51908 dst: /127.0.0.1:37399
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T16:24:54,677 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@54dc8c82{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T16:24:54,677 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1d5f8483{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-11T16:24:54,677 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-11T16:24:54,677 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6e094446{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-11T16:24:54,678 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@17d86da6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90c660c-600a-9166-c37d-1b9c62c53b37/hadoop.log.dir/,STOPPED}
2024-12-11T16:24:54,684 WARN  [BP-119074730-172.17.0.2-1733934273371 heartbeating to localhost/127.0.0.1:43789 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-11T16:24:54,684 WARN  [BP-119074730-172.17.0.2-1733934273371 heartbeating to localhost/127.0.0.1:43789 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-119074730-172.17.0.2-1733934273371 (Datanode Uuid 518c53e3-a44d-421f-a0df-e28575189a25) service to localhost/127.0.0.1:43789
2024-12-11T16:24:54,685 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90c660c-600a-9166-c37d-1b9c62c53b37/cluster_c6aa3bec-caf4-e30f-1389-b14fd52f58cf/data/data3/current/BP-119074730-172.17.0.2-1733934273371 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T16:24:54,685 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90c660c-600a-9166-c37d-1b9c62c53b37/cluster_c6aa3bec-caf4-e30f-1389-b14fd52f58cf/data/data4/current/BP-119074730-172.17.0.2-1733934273371 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T16:24:54,686 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-11T16:24:54,690 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-11T16:24:54,690 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-11T16:24:54,709 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-11T16:24:54,712 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-11T16:24:54,719 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-11T16:24:54,719 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-11T16:24:54,719 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-11T16:24:54,723 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1377bd3b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90c660c-600a-9166-c37d-1b9c62c53b37/hadoop.log.dir/,AVAILABLE}
2024-12-11T16:24:54,724 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4e952891{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-11T16:24:54,861 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1ce4d2e7{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90c660c-600a-9166-c37d-1b9c62c53b37/java.io.tmpdir/jetty-localhost-33101-hadoop-hdfs-3_4_1-tests_jar-_-any-12462243076386481488/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T16:24:54,862 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7a9b8266{HTTP/1.1, (http/1.1)}{localhost:33101}
2024-12-11T16:24:54,862 INFO  [Time-limited test {}] server.Server(415): Started @185533ms
2024-12-11T16:24:54,864 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-11T16:24:55,032 WARN  [Thread-1417 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-11T16:24:55,034 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x870aef6c0cf49337 with lease ID 0x5a0312432928716f: from storage DS-d0922336-6ec8-41bd-9084-f220c6cd95f5 node DatanodeRegistration(127.0.0.1:36515, datanodeUuid=94f6fc17-0a75-416c-8864-c6b69c7ce840, infoPort=36639, infoSecurePort=0, ipcPort=34583, storageInfo=lv=-57;cid=testClusterID;nsid=6436803;c=1733934273371), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-11T16:24:55,035 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x870aef6c0cf49337 with lease ID 0x5a0312432928716f: from storage DS-eb41716c-cf35-4f8a-a3bd-166a79ec6316 node DatanodeRegistration(127.0.0.1:36515, datanodeUuid=94f6fc17-0a75-416c-8864-c6b69c7ce840, infoPort=36639, infoSecurePort=0, ipcPort=34583, storageInfo=lv=-57;cid=testClusterID;nsid=6436803;c=1733934273371), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-11T16:24:55,243 WARN  [Thread-1438 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-11T16:24:55,246 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1e9805cf14d0d5ac with lease ID 0x5a03124329287170: from storage DS-31d80686-d824-4918-894d-bf029f453b89 node DatanodeRegistration(127.0.0.1:39199, datanodeUuid=518c53e3-a44d-421f-a0df-e28575189a25, infoPort=35185, infoSecurePort=0, ipcPort=45133, storageInfo=lv=-57;cid=testClusterID;nsid=6436803;c=1733934273371), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-11T16:24:55,246 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1e9805cf14d0d5ac with lease ID 0x5a03124329287170: from storage DS-008cd667-4edc-4c90-92f3-4613f09bac93 node DatanodeRegistration(127.0.0.1:39199, datanodeUuid=518c53e3-a44d-421f-a0df-e28575189a25, infoPort=35185, infoSecurePort=0, ipcPort=45133, storageInfo=lv=-57;cid=testClusterID;nsid=6436803;c=1733934273371), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-11T16:24:55,323 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:24:55,334 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:24:55,919 INFO  [Time-limited test {}] wal.TestLogRolling(389): Data Nodes restarted
2024-12-11T16:24:55,927 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1004
2024-12-11T16:24:55,929 ERROR [FSHLog-0-hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1-prefix:7446a65196a1,45491,1733934275250 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37399,DS-31d80686-d824-4918-894d-bf029f453b89,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:55,929 WARN  [FSHLog-0-hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1-prefix:7446a65196a1,45491,1733934275250 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37399,DS-31d80686-d824-4918-894d-bf029f453b89,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:55,931 DEBUG [regionserver/7446a65196a1:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 7446a65196a1%2C45491%2C1733934275250:(num 1733934290235) roll requested
2024-12-11T16:24:55,931 INFO  [regionserver/7446a65196a1:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7446a65196a1%2C45491%2C1733934275250.1733934295931
2024-12-11T16:24:55,973 DEBUG [regionserver/7446a65196a1:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/WALs/7446a65196a1,45491,1733934275250/7446a65196a1%2C45491%2C1733934275250.1733934290235 newFile=hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/WALs/7446a65196a1,45491,1733934275250/7446a65196a1%2C45491%2C1733934275250.1733934295931
2024-12-11T16:24:55,973 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:24:55,975 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:24:55,975 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:24:55,975 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:24:55,975 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:24:55,975 INFO  [regionserver/7446a65196a1:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/WALs/7446a65196a1,45491,1733934275250/7446a65196a1%2C45491%2C1733934275250.1733934290235 with entries=2, filesize=2.37 KB; new WAL /user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/WALs/7446a65196a1,45491,1733934275250/7446a65196a1%2C45491%2C1733934275250.1733934295931
2024-12-11T16:24:55,975 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37399,DS-31d80686-d824-4918-894d-bf029f453b89,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:55,976 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37399,DS-31d80686-d824-4918-894d-bf029f453b89,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:55,976 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/WALs/7446a65196a1,45491,1733934275250/7446a65196a1%2C45491%2C1733934275250.1733934290235
2024-12-11T16:24:55,976 WARN  [IPC Server handler 2 on default port 43789 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/WALs/7446a65196a1,45491,1733934275250/7446a65196a1%2C45491%2C1733934275250.1733934290235 has not been closed. Lease recovery is in progress. RecoveryId = 1020 for block blk_1073741837_1018
2024-12-11T16:24:55,977 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/WALs/7446a65196a1,45491,1733934275250/7446a65196a1%2C45491%2C1733934275250.1733934290235 after 1ms
2024-12-11T16:24:55,978 DEBUG [regionserver/7446a65196a1:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35185:35185),(127.0.0.1/127.0.0.1:36639:36639)]
2024-12-11T16:24:55,978 DEBUG [regionserver/7446a65196a1:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/WALs/7446a65196a1,45491,1733934275250/7446a65196a1%2C45491%2C1733934275250.1733934290235 is not closed yet, will try archiving it next time
2024-12-11T16:24:56,324 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:24:56,334 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:24:57,325 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:24:57,335 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:24:57,979 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7446a65196a1%2C45491%2C1733934275250.1733934297979
2024-12-11T16:24:58,008 DEBUG [Time-limited test {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/WALs/7446a65196a1,45491,1733934275250/7446a65196a1%2C45491%2C1733934275250.1733934295931 newFile=hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/WALs/7446a65196a1,45491,1733934275250/7446a65196a1%2C45491%2C1733934275250.1733934297979
2024-12-11T16:24:58,011 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:24:58,014 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:24:58,014 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:24:58,014 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:24:58,014 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:24:58,015 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/WALs/7446a65196a1,45491,1733934275250/7446a65196a1%2C45491%2C1733934275250.1733934295931 with entries=1, filesize=1.23 KB; new WAL /user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/WALs/7446a65196a1,45491,1733934275250/7446a65196a1%2C45491%2C1733934275250.1733934297979
2024-12-11T16:24:58,028 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35185:35185),(127.0.0.1/127.0.0.1:36639:36639)]
2024-12-11T16:24:58,028 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36515 is added to blk_1073741838_1019 (size=1264)
2024-12-11T16:24:58,028 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/WALs/7446a65196a1,45491,1733934275250/7446a65196a1%2C45491%2C1733934275250.1733934290235 is not closed yet, will try archiving it next time
2024-12-11T16:24:58,028 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/WALs/7446a65196a1,45491,1733934275250/7446a65196a1%2C45491%2C1733934275250.1733934295931 is not closed yet, will try archiving it next time
2024-12-11T16:24:58,028 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39199 is added to blk_1073741838_1019 (size=1264)
2024-12-11T16:24:58,028 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/WALs/7446a65196a1,45491,1733934275250/7446a65196a1%2C45491%2C1733934275250.1733934275895
2024-12-11T16:24:58,028 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/WALs/7446a65196a1,45491,1733934275250/7446a65196a1%2C45491%2C1733934275250.1733934275895
2024-12-11T16:24:58,029 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/WALs/7446a65196a1,45491,1733934275250/7446a65196a1%2C45491%2C1733934275250.1733934275895 after 1ms
2024-12-11T16:24:58,029 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/WALs/7446a65196a1,45491,1733934275250/7446a65196a1%2C45491%2C1733934275250.1733934275895
2024-12-11T16:24:58,035 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741837_1018: GenerationStamp not matched, existing replica is blk_1073741837_1016
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-12-11T16:24:58,042 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #3: [\x00/METAFAMILY:HBASE::REGION_EVENT::REGION_OPEN/1733934277190/Put/vlen=218/seqid=0]
2024-12-11T16:24:58,042 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #4: [row1002/info:/1733934286851/Put/vlen=1045/seqid=0]
2024-12-11T16:24:58,042 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/WALs/7446a65196a1,45491,1733934275250/7446a65196a1%2C45491%2C1733934275250.1733934275895
2024-12-11T16:24:58,042 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/WALs/7446a65196a1,45491,1733934275250/7446a65196a1%2C45491%2C1733934275250.1733934290235
2024-12-11T16:24:58,042 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/WALs/7446a65196a1,45491,1733934275250/7446a65196a1%2C45491%2C1733934275250.1733934290235
2024-12-11T16:24:58,043 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/WALs/7446a65196a1,45491,1733934275250/7446a65196a1%2C45491%2C1733934275250.1733934290235 after 1ms
2024-12-11T16:24:58,043 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/WALs/7446a65196a1,45491,1733934275250/7446a65196a1%2C45491%2C1733934275250.1733934290235
2024-12-11T16:24:58,060 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #5: [row1003/info:/1733934290234/Put/vlen=1045/seqid=0]
2024-12-11T16:24:58,060 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #6: [row1004/info:/1733934292269/Put/vlen=1045/seqid=0]
2024-12-11T16:24:58,060 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/WALs/7446a65196a1,45491,1733934275250/7446a65196a1%2C45491%2C1733934275250.1733934290235
2024-12-11T16:24:58,061 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/WALs/7446a65196a1,45491,1733934275250/7446a65196a1%2C45491%2C1733934275250.1733934295931
2024-12-11T16:24:58,061 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/WALs/7446a65196a1,45491,1733934275250/7446a65196a1%2C45491%2C1733934275250.1733934295931
2024-12-11T16:24:58,061 WARN  [IPC Server handler 2 on default port 43789 {}] namenode.FSNamesystem(3730): BLOCK* internalReleaseLease: All existing blocks are COMPLETE, lease removed, file /user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/WALs/7446a65196a1,45491,1733934275250/7446a65196a1%2C45491%2C1733934275250.1733934295931 closed.
2024-12-11T16:24:58,061 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/WALs/7446a65196a1,45491,1733934275250/7446a65196a1%2C45491%2C1733934275250.1733934295931 after 0ms
2024-12-11T16:24:58,061 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/WALs/7446a65196a1,45491,1733934275250/7446a65196a1%2C45491%2C1733934275250.1733934295931
2024-12-11T16:24:58,066 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #7: [row1005/info:/1733934295929/Put/vlen=1045/seqid=0]
2024-12-11T16:24:58,066 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/WALs/7446a65196a1,45491,1733934275250/7446a65196a1%2C45491%2C1733934275250.1733934297979
2024-12-11T16:24:58,066 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/WALs/7446a65196a1,45491,1733934275250/7446a65196a1%2C45491%2C1733934275250.1733934297979
2024-12-11T16:24:58,066 WARN  [IPC Server handler 3 on default port 43789 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/WALs/7446a65196a1,45491,1733934275250/7446a65196a1%2C45491%2C1733934275250.1733934297979 has not been closed. Lease recovery is in progress. RecoveryId = 1022 for block blk_1073741839_1021
2024-12-11T16:24:58,067 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/WALs/7446a65196a1,45491,1733934275250/7446a65196a1%2C45491%2C1733934275250.1733934297979 after 1ms
2024-12-11T16:24:58,326 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:24:58,336 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:24:58,423 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/WALs/7446a65196a1,45491,1733934275250/7446a65196a1%2C45491%2C1733934275250.1733934290235 is not closed yet, will try archiving it next time
2024-12-11T16:24:59,038 WARN  [ResponseProcessor for block BP-119074730-172.17.0.2-1733934273371:blk_1073741839_1021 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-119074730-172.17.0.2-1733934273371:blk_1073741839_1021
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:59,038 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1292506435_22 at /127.0.0.1:36636 [Receiving block BP-119074730-172.17.0.2-1733934273371:blk_1073741839_1021] {}] datanode.DataXceiver(331): 127.0.0.1:39199:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:36636 dst: /127.0.0.1:39199
java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:39199 remote=/127.0.0.1:36636]. Total timeout mills is 60000, 58969 millis timeout left.
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T16:24:59,038 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1292506435_22 at /127.0.0.1:57916 [Receiving block BP-119074730-172.17.0.2-1733934273371:blk_1073741839_1021] {}] datanode.DataXceiver(331): 127.0.0.1:36515:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:57916 dst: /127.0.0.1:36515
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T16:24:59,038 WARN  [DataStreamer for file /user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/WALs/7446a65196a1,45491,1733934275250/7446a65196a1%2C45491%2C1733934275250.1733934297979 block BP-119074730-172.17.0.2-1733934273371:blk_1073741839_1021 {}] hdfs.DataStreamer(1731): Error Recovery for BP-119074730-172.17.0.2-1733934273371:blk_1073741839_1021 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39199,DS-31d80686-d824-4918-894d-bf029f453b89,DISK], DatanodeInfoWithStorage[127.0.0.1:36515,DS-d0922336-6ec8-41bd-9084-f220c6cd95f5,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39199,DS-31d80686-d824-4918-894d-bf029f453b89,DISK]) is bad.
2024-12-11T16:24:59,042 WARN  [DataStreamer for file /user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/WALs/7446a65196a1,45491,1733934275250/7446a65196a1%2C45491%2C1733934275250.1733934297979 block BP-119074730-172.17.0.2-1733934273371:blk_1073741839_1021 {}] hdfs.DataStreamer(859): DataStreamer Exception
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-119074730-172.17.0.2-1733934273371:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:24:59,046 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39199 is added to blk_1073741839_1022 (size=85)
2024-12-11T16:24:59,326 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:24:59,337 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:24:59,978 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/WALs/7446a65196a1,45491,1733934275250/7446a65196a1%2C45491%2C1733934275250.1733934290235 after 4002ms
2024-12-11T16:25:00,327 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:00,338 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:01,328 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:01,338 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:02,072 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/WALs/7446a65196a1,45491,1733934275250/7446a65196a1%2C45491%2C1733934275250.1733934297979 after 4006ms
2024-12-11T16:25:02,072 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/WALs/7446a65196a1,45491,1733934275250/7446a65196a1%2C45491%2C1733934275250.1733934297979
2024-12-11T16:25:02,078 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/WALs/7446a65196a1,45491,1733934275250/7446a65196a1%2C45491%2C1733934275250.1733934297979
2024-12-11T16:25:02,079 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.74 KB heapSize=3.77 KB
2024-12-11T16:25:02,079 ERROR [FSHLog-0-hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1-prefix:7446a65196a1,45491,1733934275250.meta {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43085,DS-d0922336-6ec8-41bd-9084-f220c6cd95f5,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:25:02,079 WARN  [FSHLog-0-hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1-prefix:7446a65196a1,45491,1733934275250.meta {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43085,DS-d0922336-6ec8-41bd-9084-f220c6cd95f5,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:25:02,080 DEBUG [regionserver/7446a65196a1:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 7446a65196a1%2C45491%2C1733934275250.meta:.meta(num 1733934276599) roll requested
2024-12-11T16:25:02,080 INFO  [regionserver/7446a65196a1:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7446a65196a1%2C45491%2C1733934275250.meta.1733934302080.meta
2024-12-11T16:25:02,104 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:25:02,104 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:25:02,105 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:25:02,105 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:25:02,105 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:25:02,105 INFO  [regionserver/7446a65196a1:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/WALs/7446a65196a1,45491,1733934275250/7446a65196a1%2C45491%2C1733934275250.meta.1733934276599.meta with entries=8, filesize=2.36 KB; new WAL /user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/WALs/7446a65196a1,45491,1733934275250/7446a65196a1%2C45491%2C1733934275250.meta.1733934302080.meta
2024-12-11T16:25:02,111 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43085,DS-d0922336-6ec8-41bd-9084-f220c6cd95f5,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:25:02,111 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43085,DS-d0922336-6ec8-41bd-9084-f220c6cd95f5,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:25:02,111 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/WALs/7446a65196a1,45491,1733934275250/7446a65196a1%2C45491%2C1733934275250.meta.1733934276599.meta
2024-12-11T16:25:02,112 WARN  [IPC Server handler 0 on default port 43789 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/WALs/7446a65196a1,45491,1733934275250/7446a65196a1%2C45491%2C1733934275250.meta.1733934276599.meta has not been closed. Lease recovery is in progress. RecoveryId = 1024 for block blk_1073741834_1013
2024-12-11T16:25:02,112 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/WALs/7446a65196a1,45491,1733934275250/7446a65196a1%2C45491%2C1733934275250.meta.1733934276599.meta after 1ms
2024-12-11T16:25:02,129 DEBUG [regionserver/7446a65196a1:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35185:35185),(127.0.0.1/127.0.0.1:36639:36639)]
2024-12-11T16:25:02,129 DEBUG [regionserver/7446a65196a1:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/WALs/7446a65196a1,45491,1733934275250/7446a65196a1%2C45491%2C1733934275250.meta.1733934276599.meta is not closed yet, will try archiving it next time
2024-12-11T16:25:02,166 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/data/hbase/meta/1588230740/.tmp/info/8b6bffef28274f8b910b5b0b99dc98e3 is 207, key is TestLogRolling-testLogRollOnPipelineRestart,,1733934276809.455cd7920d80c3fa69ca86699047709c./info:regioninfo/1733934277195/Put/seqid=0
2024-12-11T16:25:02,204 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36515 is added to blk_1073741841_1025 (size=7125)
2024-12-11T16:25:02,204 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39199 is added to blk_1073741841_1025 (size=7125)
2024-12-11T16:25:02,207 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.52 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/data/hbase/meta/1588230740/.tmp/info/8b6bffef28274f8b910b5b0b99dc98e3
2024-12-11T16:25:02,252 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/data/hbase/meta/1588230740/.tmp/ns/d70446526f434783887b02ada62d8e90 is 43, key is default/ns:d/1733934276706/Put/seqid=0
2024-12-11T16:25:02,263 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39199 is added to blk_1073741842_1026 (size=5153)
2024-12-11T16:25:02,263 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36515 is added to blk_1073741842_1026 (size=5153)
2024-12-11T16:25:02,265 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/data/hbase/meta/1588230740/.tmp/ns/d70446526f434783887b02ada62d8e90
2024-12-11T16:25:02,310 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/data/hbase/meta/1588230740/.tmp/table/1b43a12cc43e4fcd8a9c101fa602210b is 79, key is TestLogRolling-testLogRollOnPipelineRestart/table:state/1733934277205/Put/seqid=0
2024-12-11T16:25:02,325 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36515 is added to blk_1073741843_1027 (size=5438)
2024-12-11T16:25:02,325 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39199 is added to blk_1073741843_1027 (size=5438)
2024-12-11T16:25:02,328 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:02,339 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:02,726 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=150 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/data/hbase/meta/1588230740/.tmp/table/1b43a12cc43e4fcd8a9c101fa602210b
2024-12-11T16:25:02,735 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/data/hbase/meta/1588230740/.tmp/info/8b6bffef28274f8b910b5b0b99dc98e3 as hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/data/hbase/meta/1588230740/info/8b6bffef28274f8b910b5b0b99dc98e3
2024-12-11T16:25:02,742 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/data/hbase/meta/1588230740/info/8b6bffef28274f8b910b5b0b99dc98e3, entries=10, sequenceid=11, filesize=7.0 K
2024-12-11T16:25:02,743 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/data/hbase/meta/1588230740/.tmp/ns/d70446526f434783887b02ada62d8e90 as hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/data/hbase/meta/1588230740/ns/d70446526f434783887b02ada62d8e90
2024-12-11T16:25:02,749 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/data/hbase/meta/1588230740/ns/d70446526f434783887b02ada62d8e90, entries=2, sequenceid=11, filesize=5.0 K
2024-12-11T16:25:02,750 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/data/hbase/meta/1588230740/.tmp/table/1b43a12cc43e4fcd8a9c101fa602210b as hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/data/hbase/meta/1588230740/table/1b43a12cc43e4fcd8a9c101fa602210b
2024-12-11T16:25:02,756 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/data/hbase/meta/1588230740/table/1b43a12cc43e4fcd8a9c101fa602210b, entries=2, sequenceid=11, filesize=5.3 K
2024-12-11T16:25:02,757 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.74 KB/1782, heapSize ~3.48 KB/3560, currentSize=0 B/0 for 1588230740 in 678ms, sequenceid=11, compaction requested=false
2024-12-11T16:25:02,757 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1588230740:

2024-12-11T16:25:02,758 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing 455cd7920d80c3fa69ca86699047709c 1/1 column families, dataSize=4.20 KB heapSize=4.75 KB
2024-12-11T16:25:02,758 ERROR [FSHLog-0-hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1-prefix:7446a65196a1,45491,1733934275250 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-119074730-172.17.0.2-1733934273371:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:25:02,758 WARN  [FSHLog-0-hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1-prefix:7446a65196a1,45491,1733934275250 {}] wal.AbstractFSWAL(2174): append entry failed
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-119074730-172.17.0.2-1733934273371:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:25:02,759 DEBUG [regionserver/7446a65196a1:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 7446a65196a1%2C45491%2C1733934275250:(num 1733934297979) roll requested
2024-12-11T16:25:02,759 INFO  [regionserver/7446a65196a1:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7446a65196a1%2C45491%2C1733934275250.1733934302759
2024-12-11T16:25:02,765 DEBUG [regionserver/7446a65196a1:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/WALs/7446a65196a1,45491,1733934275250/7446a65196a1%2C45491%2C1733934275250.1733934297979 newFile=hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/WALs/7446a65196a1,45491,1733934275250/7446a65196a1%2C45491%2C1733934275250.1733934302759
2024-12-11T16:25:02,765 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:25:02,765 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:25:02,765 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:25:02,765 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:25:02,766 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:25:02,766 INFO  [regionserver/7446a65196a1:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/WALs/7446a65196a1,45491,1733934275250/7446a65196a1%2C45491%2C1733934275250.1733934297979 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/WALs/7446a65196a1,45491,1733934275250/7446a65196a1%2C45491%2C1733934275250.1733934302759
2024-12-11T16:25:02,766 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-119074730-172.17.0.2-1733934273371:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:25:02,766 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-119074730-172.17.0.2-1733934273371:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:25:02,767 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/WALs/7446a65196a1,45491,1733934275250/7446a65196a1%2C45491%2C1733934275250.1733934297979
2024-12-11T16:25:02,767 DEBUG [regionserver/7446a65196a1:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36639:36639),(127.0.0.1/127.0.0.1:35185:35185)]
2024-12-11T16:25:02,767 DEBUG [regionserver/7446a65196a1:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/WALs/7446a65196a1,45491,1733934275250/7446a65196a1%2C45491%2C1733934275250.1733934297979 is not closed yet, will try archiving it next time
2024-12-11T16:25:02,767 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/WALs/7446a65196a1,45491,1733934275250/7446a65196a1%2C45491%2C1733934275250.1733934297979 after 0ms
2024-12-11T16:25:02,767 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/WALs/7446a65196a1,45491,1733934275250/7446a65196a1%2C45491%2C1733934275250.1733934297979 to hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/oldWALs/7446a65196a1%2C45491%2C1733934275250.1733934297979
2024-12-11T16:25:02,781 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/data/default/TestLogRolling-testLogRollOnPipelineRestart/455cd7920d80c3fa69ca86699047709c/.tmp/info/29119fb605114e52a8237d824777f39b is 1080, key is row1002/info:/1733934286851/Put/seqid=0
2024-12-11T16:25:02,786 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36515 is added to blk_1073741845_1029 (size=9270)
2024-12-11T16:25:02,787 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39199 is added to blk_1073741845_1029 (size=9270)
2024-12-11T16:25:03,186 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.20 KB at sequenceid=8 (bloomFilter=true), to=hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/data/default/TestLogRolling-testLogRollOnPipelineRestart/455cd7920d80c3fa69ca86699047709c/.tmp/info/29119fb605114e52a8237d824777f39b
2024-12-11T16:25:03,200 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/data/default/TestLogRolling-testLogRollOnPipelineRestart/455cd7920d80c3fa69ca86699047709c/.tmp/info/29119fb605114e52a8237d824777f39b as hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/data/default/TestLogRolling-testLogRollOnPipelineRestart/455cd7920d80c3fa69ca86699047709c/info/29119fb605114e52a8237d824777f39b
2024-12-11T16:25:03,207 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/data/default/TestLogRolling-testLogRollOnPipelineRestart/455cd7920d80c3fa69ca86699047709c/info/29119fb605114e52a8237d824777f39b, entries=4, sequenceid=8, filesize=9.1 K
2024-12-11T16:25:03,208 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~4.20 KB/4304, heapSize ~4.73 KB/4848, currentSize=0 B/0 for 455cd7920d80c3fa69ca86699047709c in 451ms, sequenceid=8, compaction requested=false
2024-12-11T16:25:03,208 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 455cd7920d80c3fa69ca86699047709c:

2024-12-11T16:25:03,214 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-12-11T16:25:03,214 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-11T16:25:03,214 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-11T16:25:03,214 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T16:25:03,214 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T16:25:03,215 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-12-11T16:25:03,215 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1395535952, stopped=false
2024-12-11T16:25:03,215 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-12-11T16:25:03,215 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=7446a65196a1,37355,1733934275106
2024-12-11T16:25:03,259 DEBUG [pool-525-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45491-0x10015bf18ae0001, quorum=127.0.0.1:50451, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-11T16:25:03,259 DEBUG [pool-525-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45491-0x10015bf18ae0001, quorum=127.0.0.1:50451, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:25:03,263 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45491-0x10015bf18ae0001, quorum=127.0.0.1:50451, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-11T16:25:03,264 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-11T16:25:03,264 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-11T16:25:03,264 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-11T16:25:03,264 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T16:25:03,264 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '7446a65196a1,45491,1733934275250' *****
2024-12-11T16:25:03,264 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-12-11T16:25:03,264 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37355-0x10015bf18ae0000, quorum=127.0.0.1:50451, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-11T16:25:03,265 INFO  [RS:0;7446a65196a1:45491 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-11T16:25:03,265 INFO  [RS:0;7446a65196a1:45491 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-11T16:25:03,265 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-12-11T16:25:03,265 INFO  [RS:0;7446a65196a1:45491 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-11T16:25:03,265 INFO  [RS:0;7446a65196a1:45491 {}] regionserver.HRegionServer(3091): Received CLOSE for 455cd7920d80c3fa69ca86699047709c
2024-12-11T16:25:03,265 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:37355-0x10015bf18ae0000, quorum=127.0.0.1:50451, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-11T16:25:03,265 INFO  [RS:0;7446a65196a1:45491 {}] regionserver.HRegionServer(959): stopping server 7446a65196a1,45491,1733934275250
2024-12-11T16:25:03,265 INFO  [RS:0;7446a65196a1:45491 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-11T16:25:03,265 INFO  [RS:0;7446a65196a1:45491 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;7446a65196a1:45491.
2024-12-11T16:25:03,265 DEBUG [RS:0;7446a65196a1:45491 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-11T16:25:03,266 DEBUG [RS:0;7446a65196a1:45491 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T16:25:03,266 INFO  [RS:0;7446a65196a1:45491 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-11T16:25:03,266 INFO  [RS:0;7446a65196a1:45491 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-11T16:25:03,266 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37355-0x10015bf18ae0000, quorum=127.0.0.1:50451, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:25:03,266 INFO  [RS:0;7446a65196a1:45491 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-11T16:25:03,266 INFO  [RS:0;7446a65196a1:45491 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-12-11T16:25:03,266 INFO  [RS:0;7446a65196a1:45491 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-12-11T16:25:03,266 DEBUG [RS:0;7446a65196a1:45491 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 455cd7920d80c3fa69ca86699047709c=TestLogRolling-testLogRollOnPipelineRestart,,1733934276809.455cd7920d80c3fa69ca86699047709c.}
2024-12-11T16:25:03,266 DEBUG [RS:0;7446a65196a1:45491 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 455cd7920d80c3fa69ca86699047709c
2024-12-11T16:25:03,266 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-11T16:25:03,267 INFO  [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-11T16:25:03,267 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-11T16:25:03,267 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-11T16:25:03,267 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-11T16:25:03,267 DEBUG [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 455cd7920d80c3fa69ca86699047709c, disabling compactions & flushes
2024-12-11T16:25:03,267 INFO  [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1733934276809.455cd7920d80c3fa69ca86699047709c.
2024-12-11T16:25:03,267 DEBUG [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1733934276809.455cd7920d80c3fa69ca86699047709c.
2024-12-11T16:25:03,267 DEBUG [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1733934276809.455cd7920d80c3fa69ca86699047709c. after waiting 0 ms
2024-12-11T16:25:03,267 DEBUG [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1733934276809.455cd7920d80c3fa69ca86699047709c.
2024-12-11T16:25:03,280 DEBUG [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/data/default/TestLogRolling-testLogRollOnPipelineRestart/455cd7920d80c3fa69ca86699047709c/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1
2024-12-11T16:25:03,280 INFO  [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnPipelineRestart,,1733934276809.455cd7920d80c3fa69ca86699047709c.
2024-12-11T16:25:03,281 DEBUG [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 455cd7920d80c3fa69ca86699047709c:
Waiting for close lock at 1733934303267Running coprocessor pre-close hooks at 1733934303267Disabling compacts and flushes for region at 1733934303267Disabling writes for close at 1733934303267Writing region close event to WAL at 1733934303267Running coprocessor post-close hooks at 1733934303280 (+13 ms)Closed at 1733934303280
2024-12-11T16:25:03,281 DEBUG [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRollOnPipelineRestart,,1733934276809.455cd7920d80c3fa69ca86699047709c.
2024-12-11T16:25:03,282 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-12-11T16:25:03,282 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-11T16:25:03,282 INFO  [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-11T16:25:03,283 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733934303266Running coprocessor pre-close hooks at 1733934303266Disabling compacts and flushes for region at 1733934303266Disabling writes for close at 1733934303267 (+1 ms)Writing region close event to WAL at 1733934303271 (+4 ms)Running coprocessor post-close hooks at 1733934303282 (+11 ms)Closed at 1733934303282
2024-12-11T16:25:03,283 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-12-11T16:25:03,329 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:03,340 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:03,466 INFO  [RS:0;7446a65196a1:45491 {}] regionserver.HRegionServer(976): stopping server 7446a65196a1,45491,1733934275250; all regions closed.
2024-12-11T16:25:03,467 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:25:03,467 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:25:03,467 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:25:03,467 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:25:03,467 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:25:03,470 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36515 is added to blk_1073741840_1023 (size=825)
2024-12-11T16:25:03,470 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39199 is added to blk_1073741840_1023 (size=825)
2024-12-11T16:25:03,763 INFO  [regionserver/7446a65196a1:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-11T16:25:03,764 INFO  [regionserver/7446a65196a1:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-12-11T16:25:03,765 INFO  [regionserver/7446a65196a1:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-12-11T16:25:04,245 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741834_1013: GenerationStamp not matched, existing replica is blk_1073741834_1010
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-12-11T16:25:04,330 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:04,341 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:05,088 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-11T16:25:05,330 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:05,342 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:05,721 WARN  [BootstrapNodeManager {}] regionserver.BootstrapNodeManager(142): failed to get live region servers from master
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=7446a65196a1:37355 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?]
	at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
	at --------Future.get--------(Unknown Source) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$Stub.getLiveRegionServers(RegionServerStatusProtos.java:17191) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.client.AsyncClusterConnectionImpl.getLiveRegionServers(AsyncClusterConnectionImpl.java:139) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 10 more
2024-12-11T16:25:06,113 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/WALs/7446a65196a1,45491,1733934275250/7446a65196a1%2C45491%2C1733934275250.meta.1733934276599.meta after 4002ms
2024-12-11T16:25:06,113 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/WALs/7446a65196a1,45491,1733934275250/7446a65196a1%2C45491%2C1733934275250.meta.1733934276599.meta to hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/oldWALs/7446a65196a1%2C45491%2C1733934275250.meta.1733934276599.meta
2024-12-11T16:25:06,116 DEBUG [RS:0;7446a65196a1:45491 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/oldWALs
2024-12-11T16:25:06,116 INFO  [RS:0;7446a65196a1:45491 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 7446a65196a1%2C45491%2C1733934275250.meta:.meta(num 1733934302080)
2024-12-11T16:25:06,117 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:25:06,117 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:25:06,117 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:25:06,117 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:25:06,117 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:25:06,119 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39199 is added to blk_1073741844_1028 (size=1162)
2024-12-11T16:25:06,119 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36515 is added to blk_1073741844_1028 (size=1162)
2024-12-11T16:25:06,123 DEBUG [RS:0;7446a65196a1:45491 {}] wal.AbstractFSWAL(1256): Moved 4 WAL file(s) to /user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/oldWALs
2024-12-11T16:25:06,123 INFO  [RS:0;7446a65196a1:45491 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 7446a65196a1%2C45491%2C1733934275250:(num 1733934302759)
2024-12-11T16:25:06,123 DEBUG [RS:0;7446a65196a1:45491 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T16:25:06,124 INFO  [RS:0;7446a65196a1:45491 {}] regionserver.LeaseManager(133): Closed leases
2024-12-11T16:25:06,124 INFO  [RS:0;7446a65196a1:45491 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-11T16:25:06,124 INFO  [RS:0;7446a65196a1:45491 {}] hbase.ChoreService(370): Chore service for: regionserver/7446a65196a1:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-12-11T16:25:06,124 INFO  [RS:0;7446a65196a1:45491 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-11T16:25:06,124 INFO  [regionserver/7446a65196a1:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-11T16:25:06,124 INFO  [RS:0;7446a65196a1:45491 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45491
2024-12-11T16:25:06,167 DEBUG [pool-525-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45491-0x10015bf18ae0001, quorum=127.0.0.1:50451, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/7446a65196a1,45491,1733934275250
2024-12-11T16:25:06,167 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37355-0x10015bf18ae0000, quorum=127.0.0.1:50451, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-11T16:25:06,167 INFO  [RS:0;7446a65196a1:45491 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-11T16:25:06,175 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [7446a65196a1,45491,1733934275250]
2024-12-11T16:25:06,183 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/7446a65196a1,45491,1733934275250 already deleted, retry=false
2024-12-11T16:25:06,184 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 7446a65196a1,45491,1733934275250 expired; onlineServers=0
2024-12-11T16:25:06,184 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '7446a65196a1,37355,1733934275106' *****
2024-12-11T16:25:06,184 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-12-11T16:25:06,184 INFO  [M:0;7446a65196a1:37355 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-11T16:25:06,184 INFO  [M:0;7446a65196a1:37355 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-11T16:25:06,184 DEBUG [M:0;7446a65196a1:37355 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-12-11T16:25:06,184 DEBUG [M:0;7446a65196a1:37355 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-12-11T16:25:06,184 DEBUG [master/7446a65196a1:0:becomeActiveMaster-HFileCleaner.large.0-1733934275618 {}] cleaner.HFileCleaner(306): Exit Thread[master/7446a65196a1:0:becomeActiveMaster-HFileCleaner.large.0-1733934275618,5,FailOnTimeoutGroup]
2024-12-11T16:25:06,184 DEBUG [master/7446a65196a1:0:becomeActiveMaster-HFileCleaner.small.0-1733934275618 {}] cleaner.HFileCleaner(306): Exit Thread[master/7446a65196a1:0:becomeActiveMaster-HFileCleaner.small.0-1733934275618,5,FailOnTimeoutGroup]
2024-12-11T16:25:06,184 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-12-11T16:25:06,184 INFO  [M:0;7446a65196a1:37355 {}] hbase.ChoreService(370): Chore service for: master/7446a65196a1:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-12-11T16:25:06,185 INFO  [M:0;7446a65196a1:37355 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-11T16:25:06,185 DEBUG [M:0;7446a65196a1:37355 {}] master.HMaster(1795): Stopping service threads
2024-12-11T16:25:06,185 INFO  [M:0;7446a65196a1:37355 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-12-11T16:25:06,185 INFO  [M:0;7446a65196a1:37355 {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-11T16:25:06,185 INFO  [M:0;7446a65196a1:37355 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-12-11T16:25:06,185 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-12-11T16:25:06,192 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37355-0x10015bf18ae0000, quorum=127.0.0.1:50451, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-12-11T16:25:06,192 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37355-0x10015bf18ae0000, quorum=127.0.0.1:50451, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:25:06,192 DEBUG [M:0;7446a65196a1:37355 {}] zookeeper.ZKUtil(347): master:37355-0x10015bf18ae0000, quorum=127.0.0.1:50451, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-12-11T16:25:06,192 WARN  [M:0;7446a65196a1:37355 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-12-11T16:25:06,193 INFO  [M:0;7446a65196a1:37355 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/.lastflushedseqids
2024-12-11T16:25:06,198 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39199 is added to blk_1073741846_1030 (size=120)
2024-12-11T16:25:06,198 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36515 is added to blk_1073741846_1030 (size=120)
2024-12-11T16:25:06,199 INFO  [M:0;7446a65196a1:37355 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-12-11T16:25:06,199 INFO  [M:0;7446a65196a1:37355 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-12-11T16:25:06,199 DEBUG [M:0;7446a65196a1:37355 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-11T16:25:06,199 INFO  [M:0;7446a65196a1:37355 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T16:25:06,199 DEBUG [M:0;7446a65196a1:37355 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T16:25:06,199 DEBUG [M:0;7446a65196a1:37355 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-11T16:25:06,199 DEBUG [M:0;7446a65196a1:37355 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T16:25:06,199 INFO  [M:0;7446a65196a1:37355 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.18 KB heapSize=29.16 KB
2024-12-11T16:25:06,199 ERROR [FSHLog-0-hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/MasterData-prefix:7446a65196a1,37355,1733934275106 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43085,DS-d0922336-6ec8-41bd-9084-f220c6cd95f5,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:25:06,200 WARN  [FSHLog-0-hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/MasterData-prefix:7446a65196a1,37355,1733934275106 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43085,DS-d0922336-6ec8-41bd-9084-f220c6cd95f5,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:25:06,200 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(198): WAL FSHLog 7446a65196a1%2C37355%2C1733934275106:(num 1733934275402) roll requested
2024-12-11T16:25:06,200 INFO  [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7446a65196a1%2C37355%2C1733934275106.1733934306200
2024-12-11T16:25:06,208 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:25:06,208 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:25:06,209 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:25:06,209 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:25:06,209 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:25:06,209 INFO  [master:store-WAL-Roller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/MasterData/WALs/7446a65196a1,37355,1733934275106/7446a65196a1%2C37355%2C1733934275106.1733934275402 with entries=53, filesize=26.63 KB; new WAL /user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/MasterData/WALs/7446a65196a1,37355,1733934275106/7446a65196a1%2C37355%2C1733934275106.1733934306200
2024-12-11T16:25:06,209 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43085,DS-d0922336-6ec8-41bd-9084-f220c6cd95f5,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:25:06,209 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43085,DS-d0922336-6ec8-41bd-9084-f220c6cd95f5,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T16:25:06,210 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/MasterData/WALs/7446a65196a1,37355,1733934275106/7446a65196a1%2C37355%2C1733934275106.1733934275402
2024-12-11T16:25:06,210 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35185:35185),(127.0.0.1/127.0.0.1:36639:36639)]
2024-12-11T16:25:06,210 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(879): hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/MasterData/WALs/7446a65196a1,37355,1733934275106/7446a65196a1%2C37355%2C1733934275106.1733934275402 is not closed yet, will try archiving it next time
2024-12-11T16:25:06,210 WARN  [IPC Server handler 0 on default port 43789 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/MasterData/WALs/7446a65196a1,37355,1733934275106/7446a65196a1%2C37355%2C1733934275106.1733934275402 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741830_1015
2024-12-11T16:25:06,210 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/MasterData/WALs/7446a65196a1,37355,1733934275106/7446a65196a1%2C37355%2C1733934275106.1733934275402 after 0ms
2024-12-11T16:25:06,225 DEBUG [M:0;7446a65196a1:37355 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1ba2d320f20547a682aa3098459dd3e5 is 82, key is hbase:meta,,1/info:regioninfo/1733934276637/Put/seqid=0
2024-12-11T16:25:06,230 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39199 is added to blk_1073741848_1033 (size=5672)
2024-12-11T16:25:06,230 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36515 is added to blk_1073741848_1033 (size=5672)
2024-12-11T16:25:06,230 INFO  [M:0;7446a65196a1:37355 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1ba2d320f20547a682aa3098459dd3e5
2024-12-11T16:25:06,248 DEBUG [M:0;7446a65196a1:37355 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/51dd053e435540f29fa132b47a69d885 is 779, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733934277210/Put/seqid=0
2024-12-11T16:25:06,253 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39199 is added to blk_1073741849_1034 (size=6119)
2024-12-11T16:25:06,253 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36515 is added to blk_1073741849_1034 (size=6119)
2024-12-11T16:25:06,253 INFO  [M:0;7446a65196a1:37355 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.58 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/51dd053e435540f29fa132b47a69d885
2024-12-11T16:25:06,272 DEBUG [M:0;7446a65196a1:37355 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/06f6957b43bc4eab9c0c81801ad290c1 is 69, key is 7446a65196a1,45491,1733934275250/rs:state/1733934275741/Put/seqid=0
2024-12-11T16:25:06,276 DEBUG [pool-525-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45491-0x10015bf18ae0001, quorum=127.0.0.1:50451, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-11T16:25:06,276 DEBUG [pool-525-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45491-0x10015bf18ae0001, quorum=127.0.0.1:50451, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-11T16:25:06,276 INFO  [RS:0;7446a65196a1:45491 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-11T16:25:06,276 INFO  [RS:0;7446a65196a1:45491 {}] regionserver.HRegionServer(1031): Exiting; stopping=7446a65196a1,45491,1733934275250; zookeeper connection closed.
2024-12-11T16:25:06,276 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@72c4a2ef {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@72c4a2ef
2024-12-11T16:25:06,276 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-12-11T16:25:06,277 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36515 is added to blk_1073741850_1035 (size=5156)
2024-12-11T16:25:06,277 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39199 is added to blk_1073741850_1035 (size=5156)
2024-12-11T16:25:06,278 INFO  [M:0;7446a65196a1:37355 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/06f6957b43bc4eab9c0c81801ad290c1
2024-12-11T16:25:06,297 DEBUG [M:0;7446a65196a1:37355 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/930373a02ced445599fedde1ccd56d2b is 52, key is load_balancer_on/state:d/1733934276799/Put/seqid=0
2024-12-11T16:25:06,301 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36515 is added to blk_1073741851_1036 (size=5056)
2024-12-11T16:25:06,301 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39199 is added to blk_1073741851_1036 (size=5056)
2024-12-11T16:25:06,302 INFO  [M:0;7446a65196a1:37355 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/930373a02ced445599fedde1ccd56d2b
2024-12-11T16:25:06,309 DEBUG [M:0;7446a65196a1:37355 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1ba2d320f20547a682aa3098459dd3e5 as hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/1ba2d320f20547a682aa3098459dd3e5
2024-12-11T16:25:06,314 INFO  [M:0;7446a65196a1:37355 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/1ba2d320f20547a682aa3098459dd3e5, entries=8, sequenceid=56, filesize=5.5 K
2024-12-11T16:25:06,316 DEBUG [M:0;7446a65196a1:37355 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/51dd053e435540f29fa132b47a69d885 as hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/51dd053e435540f29fa132b47a69d885
2024-12-11T16:25:06,325 INFO  [M:0;7446a65196a1:37355 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/51dd053e435540f29fa132b47a69d885, entries=6, sequenceid=56, filesize=6.0 K
2024-12-11T16:25:06,326 DEBUG [M:0;7446a65196a1:37355 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/06f6957b43bc4eab9c0c81801ad290c1 as hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/06f6957b43bc4eab9c0c81801ad290c1
2024-12-11T16:25:06,331 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:06,333 INFO  [M:0;7446a65196a1:37355 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/06f6957b43bc4eab9c0c81801ad290c1, entries=1, sequenceid=56, filesize=5.0 K
2024-12-11T16:25:06,334 DEBUG [M:0;7446a65196a1:37355 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/930373a02ced445599fedde1ccd56d2b as hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/930373a02ced445599fedde1ccd56d2b
2024-12-11T16:25:06,339 INFO  [M:0;7446a65196a1:37355 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/930373a02ced445599fedde1ccd56d2b, entries=1, sequenceid=56, filesize=4.9 K
2024-12-11T16:25:06,340 INFO  [M:0;7446a65196a1:37355 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.18 KB/23738, heapSize ~29.10 KB/29800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 141ms, sequenceid=56, compaction requested=false
2024-12-11T16:25:06,342 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:06,344 INFO  [M:0;7446a65196a1:37355 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T16:25:06,344 DEBUG [M:0;7446a65196a1:37355 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733934306199Disabling compacts and flushes for region at 1733934306199Disabling writes for close at 1733934306199Obtaining lock to block concurrent updates at 1733934306199Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733934306199Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23738, getHeapSize=29800, getOffHeapSize=0, getCellsCount=67 at 1733934306200 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733934306211 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733934306211Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733934306224 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733934306225 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733934306235 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733934306248 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733934306248Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733934306258 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733934306272 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733934306272Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733934306282 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733934306296 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733934306296Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@b15fe7: reopening flushed file at 1733934306308 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@354afd1e: reopening flushed file at 1733934306315 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5af7a8f0: reopening flushed file at 1733934306325 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5a39b204: reopening flushed file at 1733934306333 (+8 ms)Finished flush of dataSize ~23.18 KB/23738, heapSize ~29.10 KB/29800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 141ms, sequenceid=56, compaction requested=false at 1733934306340 (+7 ms)Writing region close event to WAL at 1733934306344 (+4 ms)Closed at 1733934306344
2024-12-11T16:25:06,344 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:25:06,344 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:25:06,344 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:25:06,344 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:25:06,345 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:25:06,346 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36515 is added to blk_1073741847_1031 (size=757)
2024-12-11T16:25:06,347 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39199 is added to blk_1073741847_1031 (size=757)
2024-12-11T16:25:07,192 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-11T16:25:07,192 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-12-11T16:25:07,192 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-12-11T16:25:07,193 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart
2024-12-11T16:25:07,331 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:07,343 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:08,281 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:25:08,281 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:25:08,295 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:25:08,295 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:25:08,295 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:25:08,295 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:25:08,295 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:25:08,296 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:25:08,299 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:25:08,299 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:25:08,299 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:25:08,301 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:25:08,306 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:25:08,306 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:25:08,332 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:08,344 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:08,809 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-11T16:25:08,810 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:25:08,810 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:25:08,811 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:25:08,811 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:25:08,824 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:25:08,824 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:25:08,824 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:25:08,825 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:25:08,825 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:25:08,825 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:25:08,828 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:25:08,828 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:25:08,828 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:25:08,830 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:25:09,333 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:09,344 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:10,211 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/MasterData/WALs/7446a65196a1,37355,1733934275106/7446a65196a1%2C37355%2C1733934275106.1733934275402 after 4001ms
2024-12-11T16:25:10,212 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/MasterData/WALs/7446a65196a1,37355,1733934275106/7446a65196a1%2C37355%2C1733934275106.1733934275402 to hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/MasterData/oldWALs/7446a65196a1%2C37355%2C1733934275106.1733934275402
2024-12-11T16:25:10,215 INFO  [WAL-Archive-0 {}] region.MasterRegionUtils(50): Moved hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/MasterData/oldWALs/7446a65196a1%2C37355%2C1733934275106.1733934275402 to hdfs://localhost:43789/user/jenkins/test-data/ffa9890a-970f-c773-a7ef-922cd363eee1/oldWALs/7446a65196a1%2C37355%2C1733934275106.1733934275402$masterlocalwal$
2024-12-11T16:25:10,215 INFO  [M:0;7446a65196a1:37355 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-12-11T16:25:10,215 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-11T16:25:10,215 INFO  [M:0;7446a65196a1:37355 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37355
2024-12-11T16:25:10,215 INFO  [M:0;7446a65196a1:37355 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-11T16:25:10,246 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741830_1015: GenerationStamp not matched, existing replica is blk_1073741830_1006
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-12-11T16:25:10,333 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:10,345 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:10,349 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37355-0x10015bf18ae0000, quorum=127.0.0.1:50451, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-11T16:25:10,350 INFO  [M:0;7446a65196a1:37355 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-11T16:25:10,350 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37355-0x10015bf18ae0000, quorum=127.0.0.1:50451, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-11T16:25:10,352 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1ce4d2e7{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T16:25:10,353 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7a9b8266{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-11T16:25:10,353 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-11T16:25:10,353 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4e952891{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-11T16:25:10,353 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1377bd3b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90c660c-600a-9166-c37d-1b9c62c53b37/hadoop.log.dir/,STOPPED}
2024-12-11T16:25:10,354 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-11T16:25:10,354 WARN  [BP-119074730-172.17.0.2-1733934273371 heartbeating to localhost/127.0.0.1:43789 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-11T16:25:10,354 WARN  [BP-119074730-172.17.0.2-1733934273371 heartbeating to localhost/127.0.0.1:43789 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-119074730-172.17.0.2-1733934273371 (Datanode Uuid 518c53e3-a44d-421f-a0df-e28575189a25) service to localhost/127.0.0.1:43789
2024-12-11T16:25:10,354 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-11T16:25:10,355 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90c660c-600a-9166-c37d-1b9c62c53b37/cluster_c6aa3bec-caf4-e30f-1389-b14fd52f58cf/data/data3/current/BP-119074730-172.17.0.2-1733934273371 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T16:25:10,355 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90c660c-600a-9166-c37d-1b9c62c53b37/cluster_c6aa3bec-caf4-e30f-1389-b14fd52f58cf/data/data4/current/BP-119074730-172.17.0.2-1733934273371 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T16:25:10,355 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-11T16:25:10,357 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@581007a9{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T16:25:10,358 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@f40aa31{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-11T16:25:10,358 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-11T16:25:10,358 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@437e5421{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-11T16:25:10,358 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2933b309{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90c660c-600a-9166-c37d-1b9c62c53b37/hadoop.log.dir/,STOPPED}
2024-12-11T16:25:10,359 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-11T16:25:10,359 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-11T16:25:10,359 WARN  [BP-119074730-172.17.0.2-1733934273371 heartbeating to localhost/127.0.0.1:43789 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-11T16:25:10,359 WARN  [BP-119074730-172.17.0.2-1733934273371 heartbeating to localhost/127.0.0.1:43789 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-119074730-172.17.0.2-1733934273371 (Datanode Uuid 94f6fc17-0a75-416c-8864-c6b69c7ce840) service to localhost/127.0.0.1:43789
2024-12-11T16:25:10,360 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90c660c-600a-9166-c37d-1b9c62c53b37/cluster_c6aa3bec-caf4-e30f-1389-b14fd52f58cf/data/data1/current/BP-119074730-172.17.0.2-1733934273371 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T16:25:10,360 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90c660c-600a-9166-c37d-1b9c62c53b37/cluster_c6aa3bec-caf4-e30f-1389-b14fd52f58cf/data/data2/current/BP-119074730-172.17.0.2-1733934273371 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T16:25:10,360 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-11T16:25:10,369 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7a25f841{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-11T16:25:10,370 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@58a42b34{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-11T16:25:10,370 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-11T16:25:10,370 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@cdcce9d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-11T16:25:10,370 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@43b12dab{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90c660c-600a-9166-c37d-1b9c62c53b37/hadoop.log.dir/,STOPPED}
2024-12-11T16:25:10,380 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-12-11T16:25:10,401 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-12-11T16:25:10,409 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnPipelineRestart Thread=179 (was 160)
Potentially hanging thread: HMaster-EventLoopGroup-10-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-32-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-33-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.4@localhost:43789
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-31-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43789
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (548703531) connection to localhost/127.0.0.1:43789 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: HMaster-EventLoopGroup-10-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:43789
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-32-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (548703531) connection to localhost/127.0.0.1:43789 from jenkins.hfs.4
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43789
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-30-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-30-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43789
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-10-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-31-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (548703531) connection to localhost/127.0.0.1:43789 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-31-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-33-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-30-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-32-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-33-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=455 (was 452) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=627 (was 638), ProcessCount=11 (was 11), AvailableMemoryMB=1134 (was 845) - AvailableMemoryMB LEAK? -
2024-12-11T16:25:10,419 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testCompactionRecordDoesntBlockRolling Thread=179, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=627, ProcessCount=11, AvailableMemoryMB=1134
2024-12-11T16:25:10,419 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-12-11T16:25:10,419 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90c660c-600a-9166-c37d-1b9c62c53b37/hadoop.log.dir so I do NOT create it in target/test-data/ffdf3259-260d-ac79-a7e6-520e991bd2d0
2024-12-11T16:25:10,419 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90c660c-600a-9166-c37d-1b9c62c53b37/hadoop.tmp.dir so I do NOT create it in target/test-data/ffdf3259-260d-ac79-a7e6-520e991bd2d0
2024-12-11T16:25:10,419 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffdf3259-260d-ac79-a7e6-520e991bd2d0/cluster_9179545b-40d5-c31c-24dd-6aa8b56461ff, deleteOnExit=true
2024-12-11T16:25:10,419 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-12-11T16:25:10,420 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffdf3259-260d-ac79-a7e6-520e991bd2d0/test.cache.data in system properties and HBase conf
2024-12-11T16:25:10,420 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffdf3259-260d-ac79-a7e6-520e991bd2d0/hadoop.tmp.dir in system properties and HBase conf
2024-12-11T16:25:10,420 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffdf3259-260d-ac79-a7e6-520e991bd2d0/hadoop.log.dir in system properties and HBase conf
2024-12-11T16:25:10,420 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffdf3259-260d-ac79-a7e6-520e991bd2d0/mapreduce.cluster.local.dir in system properties and HBase conf
2024-12-11T16:25:10,420 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffdf3259-260d-ac79-a7e6-520e991bd2d0/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-12-11T16:25:10,420 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-12-11T16:25:10,420 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-11T16:25:10,421 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffdf3259-260d-ac79-a7e6-520e991bd2d0/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-12-11T16:25:10,421 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffdf3259-260d-ac79-a7e6-520e991bd2d0/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-12-11T16:25:10,421 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffdf3259-260d-ac79-a7e6-520e991bd2d0/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-12-11T16:25:10,421 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffdf3259-260d-ac79-a7e6-520e991bd2d0/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-11T16:25:10,421 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffdf3259-260d-ac79-a7e6-520e991bd2d0/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-12-11T16:25:10,421 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffdf3259-260d-ac79-a7e6-520e991bd2d0/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-12-11T16:25:10,421 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffdf3259-260d-ac79-a7e6-520e991bd2d0/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-11T16:25:10,421 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffdf3259-260d-ac79-a7e6-520e991bd2d0/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-11T16:25:10,421 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffdf3259-260d-ac79-a7e6-520e991bd2d0/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-12-11T16:25:10,422 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffdf3259-260d-ac79-a7e6-520e991bd2d0/nfs.dump.dir in system properties and HBase conf
2024-12-11T16:25:10,422 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffdf3259-260d-ac79-a7e6-520e991bd2d0/java.io.tmpdir in system properties and HBase conf
2024-12-11T16:25:10,422 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffdf3259-260d-ac79-a7e6-520e991bd2d0/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-11T16:25:10,422 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffdf3259-260d-ac79-a7e6-520e991bd2d0/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-12-11T16:25:10,422 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffdf3259-260d-ac79-a7e6-520e991bd2d0/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-12-11T16:25:10,440 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-11T16:25:10,836 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-11T16:25:10,839 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-11T16:25:10,846 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-11T16:25:10,846 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-11T16:25:10,846 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-11T16:25:10,846 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-11T16:25:10,847 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4182d22c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffdf3259-260d-ac79-a7e6-520e991bd2d0/hadoop.log.dir/,AVAILABLE}
2024-12-11T16:25:10,847 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1cf94809{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-11T16:25:10,939 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6f8ea346{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffdf3259-260d-ac79-a7e6-520e991bd2d0/java.io.tmpdir/jetty-localhost-44111-hadoop-hdfs-3_4_1-tests_jar-_-any-15700753882766210607/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-11T16:25:10,939 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@59ed7094{HTTP/1.1, (http/1.1)}{localhost:44111}
2024-12-11T16:25:10,939 INFO  [Time-limited test {}] server.Server(415): Started @201610ms
2024-12-11T16:25:10,950 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-11T16:25:11,275 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-11T16:25:11,278 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-11T16:25:11,287 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-11T16:25:11,287 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-11T16:25:11,287 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-11T16:25:11,288 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@82bb36f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffdf3259-260d-ac79-a7e6-520e991bd2d0/hadoop.log.dir/,AVAILABLE}
2024-12-11T16:25:11,288 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5785ef07{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-11T16:25:11,334 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:11,346 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:11,379 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@17065767{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffdf3259-260d-ac79-a7e6-520e991bd2d0/java.io.tmpdir/jetty-localhost-36633-hadoop-hdfs-3_4_1-tests_jar-_-any-5431635451882972115/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T16:25:11,379 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@374344fc{HTTP/1.1, (http/1.1)}{localhost:36633}
2024-12-11T16:25:11,379 INFO  [Time-limited test {}] server.Server(415): Started @202050ms
2024-12-11T16:25:11,380 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-11T16:25:11,415 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-11T16:25:11,418 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-11T16:25:11,422 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-11T16:25:11,422 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-11T16:25:11,422 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-11T16:25:11,422 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6bc5fff1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffdf3259-260d-ac79-a7e6-520e991bd2d0/hadoop.log.dir/,AVAILABLE}
2024-12-11T16:25:11,423 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@26ab0093{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-11T16:25:11,516 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5f44ac55{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffdf3259-260d-ac79-a7e6-520e991bd2d0/java.io.tmpdir/jetty-localhost-34339-hadoop-hdfs-3_4_1-tests_jar-_-any-15122431054724721570/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T16:25:11,516 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@303cb3d3{HTTP/1.1, (http/1.1)}{localhost:34339}
2024-12-11T16:25:11,516 INFO  [Time-limited test {}] server.Server(415): Started @202187ms
2024-12-11T16:25:11,517 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-11T16:25:12,308 WARN  [Thread-1657 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffdf3259-260d-ac79-a7e6-520e991bd2d0/cluster_9179545b-40d5-c31c-24dd-6aa8b56461ff/data/data1/current/BP-1668126573-172.17.0.2-1733934310453/current, will proceed with Du for space computation calculation, 
2024-12-11T16:25:12,309 WARN  [Thread-1658 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffdf3259-260d-ac79-a7e6-520e991bd2d0/cluster_9179545b-40d5-c31c-24dd-6aa8b56461ff/data/data2/current/BP-1668126573-172.17.0.2-1733934310453/current, will proceed with Du for space computation calculation, 
2024-12-11T16:25:12,335 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:12,346 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:12,350 WARN  [Thread-1621 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-11T16:25:12,353 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1f5c57a26b8a58c0 with lease ID 0x4f601ff06d0e6a0e: Processing first storage report for DS-71d5d51e-eb4b-4191-8123-32e050340d46 from datanode DatanodeRegistration(127.0.0.1:44201, datanodeUuid=784914ec-be2b-45eb-98b4-d953662031e6, infoPort=40679, infoSecurePort=0, ipcPort=44047, storageInfo=lv=-57;cid=testClusterID;nsid=1344086209;c=1733934310453)
2024-12-11T16:25:12,353 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1f5c57a26b8a58c0 with lease ID 0x4f601ff06d0e6a0e: from storage DS-71d5d51e-eb4b-4191-8123-32e050340d46 node DatanodeRegistration(127.0.0.1:44201, datanodeUuid=784914ec-be2b-45eb-98b4-d953662031e6, infoPort=40679, infoSecurePort=0, ipcPort=44047, storageInfo=lv=-57;cid=testClusterID;nsid=1344086209;c=1733934310453), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-11T16:25:12,353 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1f5c57a26b8a58c0 with lease ID 0x4f601ff06d0e6a0e: Processing first storage report for DS-6746fadf-7120-461f-ab6a-ad7a9b94df0d from datanode DatanodeRegistration(127.0.0.1:44201, datanodeUuid=784914ec-be2b-45eb-98b4-d953662031e6, infoPort=40679, infoSecurePort=0, ipcPort=44047, storageInfo=lv=-57;cid=testClusterID;nsid=1344086209;c=1733934310453)
2024-12-11T16:25:12,353 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1f5c57a26b8a58c0 with lease ID 0x4f601ff06d0e6a0e: from storage DS-6746fadf-7120-461f-ab6a-ad7a9b94df0d node DatanodeRegistration(127.0.0.1:44201, datanodeUuid=784914ec-be2b-45eb-98b4-d953662031e6, infoPort=40679, infoSecurePort=0, ipcPort=44047, storageInfo=lv=-57;cid=testClusterID;nsid=1344086209;c=1733934310453), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-11T16:25:12,680 WARN  [Thread-1668 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffdf3259-260d-ac79-a7e6-520e991bd2d0/cluster_9179545b-40d5-c31c-24dd-6aa8b56461ff/data/data3/current/BP-1668126573-172.17.0.2-1733934310453/current, will proceed with Du for space computation calculation, 
2024-12-11T16:25:12,681 WARN  [Thread-1669 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffdf3259-260d-ac79-a7e6-520e991bd2d0/cluster_9179545b-40d5-c31c-24dd-6aa8b56461ff/data/data4/current/BP-1668126573-172.17.0.2-1733934310453/current, will proceed with Du for space computation calculation, 
2024-12-11T16:25:12,711 WARN  [Thread-1644 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-11T16:25:12,724 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xbabcd40b65a466bc with lease ID 0x4f601ff06d0e6a0f: Processing first storage report for DS-bc86a639-4d96-490b-b020-906c1c00b22a from datanode DatanodeRegistration(127.0.0.1:44051, datanodeUuid=3aee8323-9ec0-4965-ac7c-a730bbf84d01, infoPort=36963, infoSecurePort=0, ipcPort=39083, storageInfo=lv=-57;cid=testClusterID;nsid=1344086209;c=1733934310453)
2024-12-11T16:25:12,724 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbabcd40b65a466bc with lease ID 0x4f601ff06d0e6a0f: from storage DS-bc86a639-4d96-490b-b020-906c1c00b22a node DatanodeRegistration(127.0.0.1:44051, datanodeUuid=3aee8323-9ec0-4965-ac7c-a730bbf84d01, infoPort=36963, infoSecurePort=0, ipcPort=39083, storageInfo=lv=-57;cid=testClusterID;nsid=1344086209;c=1733934310453), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-11T16:25:12,725 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xbabcd40b65a466bc with lease ID 0x4f601ff06d0e6a0f: Processing first storage report for DS-fabd4656-5141-443a-b6eb-5f72c3fabb50 from datanode DatanodeRegistration(127.0.0.1:44051, datanodeUuid=3aee8323-9ec0-4965-ac7c-a730bbf84d01, infoPort=36963, infoSecurePort=0, ipcPort=39083, storageInfo=lv=-57;cid=testClusterID;nsid=1344086209;c=1733934310453)
2024-12-11T16:25:12,725 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbabcd40b65a466bc with lease ID 0x4f601ff06d0e6a0f: from storage DS-fabd4656-5141-443a-b6eb-5f72c3fabb50 node DatanodeRegistration(127.0.0.1:44051, datanodeUuid=3aee8323-9ec0-4965-ac7c-a730bbf84d01, infoPort=36963, infoSecurePort=0, ipcPort=39083, storageInfo=lv=-57;cid=testClusterID;nsid=1344086209;c=1733934310453), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-12-11T16:25:12,761 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffdf3259-260d-ac79-a7e6-520e991bd2d0
2024-12-11T16:25:12,776 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffdf3259-260d-ac79-a7e6-520e991bd2d0/cluster_9179545b-40d5-c31c-24dd-6aa8b56461ff/zookeeper_0, clientPort=54728, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffdf3259-260d-ac79-a7e6-520e991bd2d0/cluster_9179545b-40d5-c31c-24dd-6aa8b56461ff/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffdf3259-260d-ac79-a7e6-520e991bd2d0/cluster_9179545b-40d5-c31c-24dd-6aa8b56461ff/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-11T16:25:12,777 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=54728
2024-12-11T16:25:12,777 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T16:25:12,779 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T16:25:12,800 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44201 is added to blk_1073741825_1001 (size=7)
2024-12-11T16:25:12,801 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44051 is added to blk_1073741825_1001 (size=7)
2024-12-11T16:25:12,803 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac with version=8
2024-12-11T16:25:12,803 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/hbase-staging
2024-12-11T16:25:12,806 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/7446a65196a1:0 server-side Connection retries=45
2024-12-11T16:25:12,806 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-11T16:25:12,806 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-11T16:25:12,806 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-11T16:25:12,806 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-11T16:25:12,806 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-11T16:25:12,806 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-12-11T16:25:12,806 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-11T16:25:12,807 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35189
2024-12-11T16:25:12,809 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:35189 connecting to ZooKeeper ensemble=127.0.0.1:54728
2024-12-11T16:25:13,017 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:351890x0, quorum=127.0.0.1:54728, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-11T16:25:13,018 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:35189-0x10015bfabe30000 connected
2024-12-11T16:25:13,128 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T16:25:13,130 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T16:25:13,141 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35189-0x10015bfabe30000, quorum=127.0.0.1:54728, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-11T16:25:13,141 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac, hbase.cluster.distributed=false
2024-12-11T16:25:13,143 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35189-0x10015bfabe30000, quorum=127.0.0.1:54728, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-11T16:25:13,155 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35189
2024-12-11T16:25:13,159 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35189
2024-12-11T16:25:13,175 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35189
2024-12-11T16:25:13,188 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35189
2024-12-11T16:25:13,198 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35189
2024-12-11T16:25:13,219 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/7446a65196a1:0 server-side Connection retries=45
2024-12-11T16:25:13,219 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-11T16:25:13,219 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-11T16:25:13,219 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-11T16:25:13,219 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-11T16:25:13,219 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-11T16:25:13,219 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-11T16:25:13,219 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-11T16:25:13,226 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35865
2024-12-11T16:25:13,229 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35865 connecting to ZooKeeper ensemble=127.0.0.1:54728
2024-12-11T16:25:13,230 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T16:25:13,232 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T16:25:13,244 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:358650x0, quorum=127.0.0.1:54728, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-11T16:25:13,245 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35865-0x10015bfabe30001 connected
2024-12-11T16:25:13,248 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35865-0x10015bfabe30001, quorum=127.0.0.1:54728, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-11T16:25:13,248 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-11T16:25:13,255 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-11T16:25:13,256 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35865-0x10015bfabe30001, quorum=127.0.0.1:54728, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-11T16:25:13,262 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35865-0x10015bfabe30001, quorum=127.0.0.1:54728, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-11T16:25:13,275 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35865
2024-12-11T16:25:13,275 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35865
2024-12-11T16:25:13,291 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35865
2024-12-11T16:25:13,295 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35865
2024-12-11T16:25:13,299 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35865
2024-12-11T16:25:13,326 DEBUG [M:0;7446a65196a1:35189 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;7446a65196a1:35189
2024-12-11T16:25:13,326 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/7446a65196a1,35189,1733934312805
2024-12-11T16:25:13,336 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35189-0x10015bfabe30000, quorum=127.0.0.1:54728, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-11T16:25:13,336 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35865-0x10015bfabe30001, quorum=127.0.0.1:54728, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-11T16:25:13,336 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:13,337 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35189-0x10015bfabe30000, quorum=127.0.0.1:54728, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/7446a65196a1,35189,1733934312805
2024-12-11T16:25:13,344 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35865-0x10015bfabe30001, quorum=127.0.0.1:54728, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-11T16:25:13,344 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35865-0x10015bfabe30001, quorum=127.0.0.1:54728, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:25:13,344 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35189-0x10015bfabe30000, quorum=127.0.0.1:54728, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:25:13,346 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35189-0x10015bfabe30000, quorum=127.0.0.1:54728, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-11T16:25:13,347 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:13,348 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/7446a65196a1,35189,1733934312805 from backup master directory
2024-12-11T16:25:13,360 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35189-0x10015bfabe30000, quorum=127.0.0.1:54728, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/7446a65196a1,35189,1733934312805
2024-12-11T16:25:13,360 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35865-0x10015bfabe30001, quorum=127.0.0.1:54728, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-11T16:25:13,360 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35189-0x10015bfabe30000, quorum=127.0.0.1:54728, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-11T16:25:13,360 WARN  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-11T16:25:13,360 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=7446a65196a1,35189,1733934312805
2024-12-11T16:25:13,371 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/hbase.id] with ID: 7161a193-ea3b-4ed8-9e7d-95011ec824be
2024-12-11T16:25:13,372 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/.tmp/hbase.id
2024-12-11T16:25:13,378 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44051 is added to blk_1073741826_1002 (size=42)
2024-12-11T16:25:13,378 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44201 is added to blk_1073741826_1002 (size=42)
2024-12-11T16:25:13,379 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/.tmp/hbase.id]:[hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/hbase.id]
2024-12-11T16:25:13,393 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T16:25:13,393 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-12-11T16:25:13,394 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms.
2024-12-11T16:25:13,402 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35865-0x10015bfabe30001, quorum=127.0.0.1:54728, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:25:13,402 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35189-0x10015bfabe30000, quorum=127.0.0.1:54728, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:25:13,409 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44051 is added to blk_1073741827_1003 (size=196)
2024-12-11T16:25:13,409 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44201 is added to blk_1073741827_1003 (size=196)
2024-12-11T16:25:13,410 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-11T16:25:13,410 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-11T16:25:13,411 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-11T16:25:13,425 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44051 is added to blk_1073741828_1004 (size=1189)
2024-12-11T16:25:13,425 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44201 is added to blk_1073741828_1004 (size=1189)
2024-12-11T16:25:13,426 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/MasterData/data/master/store
2024-12-11T16:25:13,436 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44051 is added to blk_1073741829_1005 (size=34)
2024-12-11T16:25:13,437 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T16:25:13,437 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-11T16:25:13,437 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T16:25:13,437 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T16:25:13,437 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-11T16:25:13,438 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T16:25:13,438 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T16:25:13,438 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733934313437Disabling compacts and flushes for region at 1733934313437Disabling writes for close at 1733934313438 (+1 ms)Writing region close event to WAL at 1733934313438Closed at 1733934313438
2024-12-11T16:25:13,438 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44201 is added to blk_1073741829_1005 (size=34)
2024-12-11T16:25:13,439 WARN  [master/7446a65196a1:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/MasterData/data/master/store/.initializing
2024-12-11T16:25:13,439 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/MasterData/WALs/7446a65196a1,35189,1733934312805
2024-12-11T16:25:13,442 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7446a65196a1%2C35189%2C1733934312805, suffix=, logDir=hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/MasterData/WALs/7446a65196a1,35189,1733934312805, archiveDir=hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/MasterData/oldWALs, maxLogs=10
2024-12-11T16:25:13,442 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7446a65196a1%2C35189%2C1733934312805.1733934313442
2024-12-11T16:25:13,455 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/MasterData/WALs/7446a65196a1,35189,1733934312805/7446a65196a1%2C35189%2C1733934312805.1733934313442
2024-12-11T16:25:13,462 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40679:40679),(127.0.0.1/127.0.0.1:36963:36963)]
2024-12-11T16:25:13,464 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-11T16:25:13,464 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T16:25:13,465 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T16:25:13,465 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T16:25:13,466 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-11T16:25:13,467 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-11T16:25:13,467 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:25:13,468 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T16:25:13,468 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-11T16:25:13,469 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-11T16:25:13,470 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:25:13,473 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-11T16:25:13,473 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-11T16:25:13,475 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-11T16:25:13,475 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:25:13,476 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-11T16:25:13,476 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-11T16:25:13,477 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-11T16:25:13,477 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:25:13,478 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-11T16:25:13,478 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T16:25:13,479 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-11T16:25:13,479 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-11T16:25:13,480 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T16:25:13,481 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T16:25:13,481 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-11T16:25:13,482 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T16:25:13,485 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-11T16:25:13,486 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=722546, jitterRate=-0.08123598992824554}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-11T16:25:13,486 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1733934313465Initializing all the Stores at 1733934313465Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733934313466 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733934313466Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733934313466Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733934313466Cleaning up temporary data from old regions at 1733934313481 (+15 ms)Region opened successfully at 1733934313486 (+5 ms)
2024-12-11T16:25:13,486 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-11T16:25:13,490 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@637498d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7446a65196a1/172.17.0.2:0
2024-12-11T16:25:13,491 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-12-11T16:25:13,491 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-11T16:25:13,491 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-11T16:25:13,491 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-11T16:25:13,491 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-12-11T16:25:13,492 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-12-11T16:25:13,492 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-11T16:25:13,494 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-11T16:25:13,494 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35189-0x10015bfabe30000, quorum=127.0.0.1:54728, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-11T16:25:13,519 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-12-11T16:25:13,520 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-11T16:25:13,522 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35189-0x10015bfabe30000, quorum=127.0.0.1:54728, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-11T16:25:13,527 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-12-11T16:25:13,528 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-11T16:25:13,529 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35189-0x10015bfabe30000, quorum=127.0.0.1:54728, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-11T16:25:13,536 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-12-11T16:25:13,537 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35189-0x10015bfabe30000, quorum=127.0.0.1:54728, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-11T16:25:13,544 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-12-11T16:25:13,546 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35189-0x10015bfabe30000, quorum=127.0.0.1:54728, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-11T16:25:13,552 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-11T16:25:13,561 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35865-0x10015bfabe30001, quorum=127.0.0.1:54728, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-11T16:25:13,561 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35189-0x10015bfabe30000, quorum=127.0.0.1:54728, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-11T16:25:13,561 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35865-0x10015bfabe30001, quorum=127.0.0.1:54728, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:25:13,561 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35189-0x10015bfabe30000, quorum=127.0.0.1:54728, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:25:13,561 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=7446a65196a1,35189,1733934312805, sessionid=0x10015bfabe30000, setting cluster-up flag (Was=false)
2024-12-11T16:25:13,577 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35865-0x10015bfabe30001, quorum=127.0.0.1:54728, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:25:13,577 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35189-0x10015bfabe30000, quorum=127.0.0.1:54728, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:25:13,602 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-11T16:25:13,604 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=7446a65196a1,35189,1733934312805
2024-12-11T16:25:13,619 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35189-0x10015bfabe30000, quorum=127.0.0.1:54728, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:25:13,619 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35865-0x10015bfabe30001, quorum=127.0.0.1:54728, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:25:13,644 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-11T16:25:13,645 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=7446a65196a1,35189,1733934312805
2024-12-11T16:25:13,647 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-12-11T16:25:13,648 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-12-11T16:25:13,649 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-12-11T16:25:13,649 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-11T16:25:13,649 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 7446a65196a1,35189,1733934312805
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-11T16:25:13,650 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/7446a65196a1:0, corePoolSize=5, maxPoolSize=5
2024-12-11T16:25:13,651 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/7446a65196a1:0, corePoolSize=5, maxPoolSize=5
2024-12-11T16:25:13,651 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/7446a65196a1:0, corePoolSize=5, maxPoolSize=5
2024-12-11T16:25:13,651 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/7446a65196a1:0, corePoolSize=5, maxPoolSize=5
2024-12-11T16:25:13,651 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/7446a65196a1:0, corePoolSize=10, maxPoolSize=10
2024-12-11T16:25:13,651 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:25:13,651 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/7446a65196a1:0, corePoolSize=2, maxPoolSize=2
2024-12-11T16:25:13,651 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:25:13,651 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733934343651
2024-12-11T16:25:13,652 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-11T16:25:13,652 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-11T16:25:13,652 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-11T16:25:13,652 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-11T16:25:13,652 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-11T16:25:13,652 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-11T16:25:13,652 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-11T16:25:13,652 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-11T16:25:13,652 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-11T16:25:13,652 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-11T16:25:13,653 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-11T16:25:13,653 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-11T16:25:13,653 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-12-11T16:25:13,653 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-11T16:25:13,653 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/7446a65196a1:0:becomeActiveMaster-HFileCleaner.large.0-1733934313653,5,FailOnTimeoutGroup]
2024-12-11T16:25:13,653 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/7446a65196a1:0:becomeActiveMaster-HFileCleaner.small.0-1733934313653,5,FailOnTimeoutGroup]
2024-12-11T16:25:13,653 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-11T16:25:13,654 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-11T16:25:13,654 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-11T16:25:13,654 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-11T16:25:13,654 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:25:13,654 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-11T16:25:13,671 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44051 is added to blk_1073741831_1007 (size=1321)
2024-12-11T16:25:13,671 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44201 is added to blk_1073741831_1007 (size=1321)
2024-12-11T16:25:13,672 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-12-11T16:25:13,672 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac
2024-12-11T16:25:13,684 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44201 is added to blk_1073741832_1008 (size=32)
2024-12-11T16:25:13,684 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44051 is added to blk_1073741832_1008 (size=32)
2024-12-11T16:25:13,706 INFO  [RS:0;7446a65196a1:35865 {}] regionserver.HRegionServer(746): ClusterId : 7161a193-ea3b-4ed8-9e7d-95011ec824be
2024-12-11T16:25:13,706 DEBUG [RS:0;7446a65196a1:35865 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-11T16:25:13,727 DEBUG [RS:0;7446a65196a1:35865 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-11T16:25:13,727 DEBUG [RS:0;7446a65196a1:35865 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-11T16:25:13,737 DEBUG [RS:0;7446a65196a1:35865 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-11T16:25:13,737 DEBUG [RS:0;7446a65196a1:35865 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7798c9b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7446a65196a1/172.17.0.2:0
2024-12-11T16:25:13,748 DEBUG [RS:0;7446a65196a1:35865 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;7446a65196a1:35865
2024-12-11T16:25:13,748 INFO  [RS:0;7446a65196a1:35865 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-12-11T16:25:13,748 INFO  [RS:0;7446a65196a1:35865 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-12-11T16:25:13,748 DEBUG [RS:0;7446a65196a1:35865 {}] regionserver.HRegionServer(832): About to register with Master.
2024-12-11T16:25:13,749 INFO  [RS:0;7446a65196a1:35865 {}] regionserver.HRegionServer(2659): reportForDuty to master=7446a65196a1,35189,1733934312805 with port=35865, startcode=1733934313218
2024-12-11T16:25:13,749 DEBUG [RS:0;7446a65196a1:35865 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-11T16:25:13,751 INFO  [HMaster-EventLoopGroup-12-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34443, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.5 (auth:SIMPLE), service=RegionServerStatusService
2024-12-11T16:25:13,752 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35189 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 7446a65196a1,35865,1733934313218
2024-12-11T16:25:13,752 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35189 {}] master.ServerManager(517): Registering regionserver=7446a65196a1,35865,1733934313218
2024-12-11T16:25:13,754 DEBUG [RS:0;7446a65196a1:35865 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac
2024-12-11T16:25:13,754 DEBUG [RS:0;7446a65196a1:35865 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43205
2024-12-11T16:25:13,754 DEBUG [RS:0;7446a65196a1:35865 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-12-11T16:25:13,761 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35189-0x10015bfabe30000, quorum=127.0.0.1:54728, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-11T16:25:13,761 DEBUG [RS:0;7446a65196a1:35865 {}] zookeeper.ZKUtil(111): regionserver:35865-0x10015bfabe30001, quorum=127.0.0.1:54728, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/7446a65196a1,35865,1733934313218
2024-12-11T16:25:13,761 WARN  [RS:0;7446a65196a1:35865 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-11T16:25:13,761 INFO  [RS:0;7446a65196a1:35865 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-11T16:25:13,761 DEBUG [RS:0;7446a65196a1:35865 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/WALs/7446a65196a1,35865,1733934313218
2024-12-11T16:25:13,764 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [7446a65196a1,35865,1733934313218]
2024-12-11T16:25:13,766 INFO  [RS:0;7446a65196a1:35865 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-11T16:25:13,768 INFO  [RS:0;7446a65196a1:35865 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-11T16:25:13,769 INFO  [RS:0;7446a65196a1:35865 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-11T16:25:13,769 INFO  [RS:0;7446a65196a1:35865 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-11T16:25:13,769 INFO  [RS:0;7446a65196a1:35865 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-12-11T16:25:13,770 INFO  [RS:0;7446a65196a1:35865 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-12-11T16:25:13,770 INFO  [RS:0;7446a65196a1:35865 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-11T16:25:13,770 DEBUG [RS:0;7446a65196a1:35865 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:25:13,770 DEBUG [RS:0;7446a65196a1:35865 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:25:13,770 DEBUG [RS:0;7446a65196a1:35865 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:25:13,770 DEBUG [RS:0;7446a65196a1:35865 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:25:13,770 DEBUG [RS:0;7446a65196a1:35865 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:25:13,771 DEBUG [RS:0;7446a65196a1:35865 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/7446a65196a1:0, corePoolSize=2, maxPoolSize=2
2024-12-11T16:25:13,771 DEBUG [RS:0;7446a65196a1:35865 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:25:13,771 DEBUG [RS:0;7446a65196a1:35865 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:25:13,771 DEBUG [RS:0;7446a65196a1:35865 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:25:13,771 DEBUG [RS:0;7446a65196a1:35865 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:25:13,771 DEBUG [RS:0;7446a65196a1:35865 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:25:13,771 DEBUG [RS:0;7446a65196a1:35865 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:25:13,771 DEBUG [RS:0;7446a65196a1:35865 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/7446a65196a1:0, corePoolSize=3, maxPoolSize=3
2024-12-11T16:25:13,771 DEBUG [RS:0;7446a65196a1:35865 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/7446a65196a1:0, corePoolSize=3, maxPoolSize=3
2024-12-11T16:25:13,772 INFO  [RS:0;7446a65196a1:35865 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-11T16:25:13,773 INFO  [RS:0;7446a65196a1:35865 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-11T16:25:13,773 INFO  [RS:0;7446a65196a1:35865 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-11T16:25:13,773 INFO  [RS:0;7446a65196a1:35865 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-11T16:25:13,773 INFO  [RS:0;7446a65196a1:35865 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-11T16:25:13,773 INFO  [RS:0;7446a65196a1:35865 {}] hbase.ChoreService(168): Chore ScheduledChore name=7446a65196a1,35865,1733934313218-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-11T16:25:13,792 INFO  [RS:0;7446a65196a1:35865 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-11T16:25:13,792 INFO  [RS:0;7446a65196a1:35865 {}] hbase.ChoreService(168): Chore ScheduledChore name=7446a65196a1,35865,1733934313218-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-11T16:25:13,792 INFO  [RS:0;7446a65196a1:35865 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T16:25:13,792 INFO  [RS:0;7446a65196a1:35865 {}] regionserver.Replication(171): 7446a65196a1,35865,1733934313218 started
2024-12-11T16:25:13,809 INFO  [RS:0;7446a65196a1:35865 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T16:25:13,809 INFO  [RS:0;7446a65196a1:35865 {}] regionserver.HRegionServer(1482): Serving as 7446a65196a1,35865,1733934313218, RpcServer on 7446a65196a1/172.17.0.2:35865, sessionid=0x10015bfabe30001
2024-12-11T16:25:13,809 DEBUG [RS:0;7446a65196a1:35865 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-11T16:25:13,809 DEBUG [RS:0;7446a65196a1:35865 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 7446a65196a1,35865,1733934313218
2024-12-11T16:25:13,809 DEBUG [RS:0;7446a65196a1:35865 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7446a65196a1,35865,1733934313218'
2024-12-11T16:25:13,809 DEBUG [RS:0;7446a65196a1:35865 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-11T16:25:13,810 DEBUG [RS:0;7446a65196a1:35865 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-11T16:25:13,810 DEBUG [RS:0;7446a65196a1:35865 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-11T16:25:13,810 DEBUG [RS:0;7446a65196a1:35865 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-11T16:25:13,810 DEBUG [RS:0;7446a65196a1:35865 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 7446a65196a1,35865,1733934313218
2024-12-11T16:25:13,810 DEBUG [RS:0;7446a65196a1:35865 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7446a65196a1,35865,1733934313218'
2024-12-11T16:25:13,810 DEBUG [RS:0;7446a65196a1:35865 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-11T16:25:13,811 DEBUG [RS:0;7446a65196a1:35865 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-11T16:25:13,811 DEBUG [RS:0;7446a65196a1:35865 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-11T16:25:13,811 INFO  [RS:0;7446a65196a1:35865 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-11T16:25:13,811 INFO  [RS:0;7446a65196a1:35865 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-11T16:25:13,913 INFO  [RS:0;7446a65196a1:35865 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7446a65196a1%2C35865%2C1733934313218, suffix=, logDir=hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/WALs/7446a65196a1,35865,1733934313218, archiveDir=hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/oldWALs, maxLogs=32
2024-12-11T16:25:13,914 INFO  [RS:0;7446a65196a1:35865 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7446a65196a1%2C35865%2C1733934313218.1733934313914
2024-12-11T16:25:13,920 INFO  [RS:0;7446a65196a1:35865 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/WALs/7446a65196a1,35865,1733934313218/7446a65196a1%2C35865%2C1733934313218.1733934313914
2024-12-11T16:25:13,930 DEBUG [RS:0;7446a65196a1:35865 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40679:40679),(127.0.0.1/127.0.0.1:36963:36963)]
2024-12-11T16:25:14,085 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T16:25:14,087 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-11T16:25:14,088 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-11T16:25:14,088 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:25:14,089 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T16:25:14,089 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-11T16:25:14,091 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-11T16:25:14,091 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:25:14,091 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T16:25:14,092 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-11T16:25:14,093 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-11T16:25:14,093 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:25:14,094 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T16:25:14,094 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-11T16:25:14,096 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-11T16:25:14,096 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:25:14,096 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T16:25:14,096 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-11T16:25:14,097 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/data/hbase/meta/1588230740
2024-12-11T16:25:14,097 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/data/hbase/meta/1588230740
2024-12-11T16:25:14,099 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-11T16:25:14,099 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-11T16:25:14,100 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-11T16:25:14,101 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-11T16:25:14,105 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-11T16:25:14,105 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=795591, jitterRate=0.0116472989320755}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-11T16:25:14,106 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1733934314085Initializing all the Stores at 1733934314086 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733934314086Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733934314087 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733934314087Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733934314087Cleaning up temporary data from old regions at 1733934314099 (+12 ms)Region opened successfully at 1733934314106 (+7 ms)
2024-12-11T16:25:14,106 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-11T16:25:14,106 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-11T16:25:14,106 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-11T16:25:14,106 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-11T16:25:14,106 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-11T16:25:14,107 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-11T16:25:14,107 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733934314106Disabling compacts and flushes for region at 1733934314106Disabling writes for close at 1733934314106Writing region close event to WAL at 1733934314106Closed at 1733934314106
2024-12-11T16:25:14,108 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-11T16:25:14,108 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-12-11T16:25:14,108 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-11T16:25:14,110 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-11T16:25:14,111 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-11T16:25:14,261 DEBUG [7446a65196a1:35189 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-12-11T16:25:14,262 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=7446a65196a1,35865,1733934313218
2024-12-11T16:25:14,263 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7446a65196a1,35865,1733934313218, state=OPENING
2024-12-11T16:25:14,294 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-11T16:25:14,337 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:14,340 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35865-0x10015bfabe30001, quorum=127.0.0.1:54728, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:25:14,340 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35189-0x10015bfabe30000, quorum=127.0.0.1:54728, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:25:14,341 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-11T16:25:14,341 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-11T16:25:14,341 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-11T16:25:14,342 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=7446a65196a1,35865,1733934313218}]
2024-12-11T16:25:14,348 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:14,496 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-11T16:25:14,498 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38519, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-11T16:25:14,501 INFO  [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-12-11T16:25:14,501 INFO  [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-11T16:25:14,503 INFO  [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7446a65196a1%2C35865%2C1733934313218.meta, suffix=.meta, logDir=hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/WALs/7446a65196a1,35865,1733934313218, archiveDir=hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/oldWALs, maxLogs=32
2024-12-11T16:25:14,504 INFO  [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 7446a65196a1%2C35865%2C1733934313218.meta.1733934314503.meta
2024-12-11T16:25:14,510 INFO  [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/WALs/7446a65196a1,35865,1733934313218/7446a65196a1%2C35865%2C1733934313218.meta.1733934314503.meta
2024-12-11T16:25:14,517 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36963:36963),(127.0.0.1/127.0.0.1:40679:40679)]
2024-12-11T16:25:14,520 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-11T16:25:14,521 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-11T16:25:14,521 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-11T16:25:14,521 INFO  [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-11T16:25:14,521 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-11T16:25:14,521 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T16:25:14,521 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-12-11T16:25:14,521 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-12-11T16:25:14,522 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-11T16:25:14,523 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-11T16:25:14,523 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:25:14,523 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T16:25:14,524 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-11T16:25:14,524 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-11T16:25:14,524 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:25:14,525 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T16:25:14,525 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-11T16:25:14,525 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-11T16:25:14,525 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:25:14,525 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T16:25:14,526 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-11T16:25:14,526 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-11T16:25:14,526 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:25:14,527 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T16:25:14,527 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-11T16:25:14,527 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/data/hbase/meta/1588230740
2024-12-11T16:25:14,529 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/data/hbase/meta/1588230740
2024-12-11T16:25:14,530 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-11T16:25:14,530 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-11T16:25:14,530 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-11T16:25:14,531 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-11T16:25:14,532 INFO  [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=703831, jitterRate=-0.10503329336643219}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-11T16:25:14,532 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-12-11T16:25:14,532 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1733934314521Writing region info on filesystem at 1733934314521Initializing all the Stores at 1733934314522 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733934314522Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733934314522Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733934314522Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733934314522Cleaning up temporary data from old regions at 1733934314530 (+8 ms)Running coprocessor post-open hooks at 1733934314532 (+2 ms)Region opened successfully at 1733934314532
2024-12-11T16:25:14,533 INFO  [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733934314496
2024-12-11T16:25:14,535 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-11T16:25:14,536 INFO  [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-12-11T16:25:14,536 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=7446a65196a1,35865,1733934313218
2024-12-11T16:25:14,537 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7446a65196a1,35865,1733934313218, state=OPEN
2024-12-11T16:25:14,612 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35865-0x10015bfabe30001, quorum=127.0.0.1:54728, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-11T16:25:14,612 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35189-0x10015bfabe30000, quorum=127.0.0.1:54728, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-11T16:25:14,612 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=7446a65196a1,35865,1733934313218
2024-12-11T16:25:14,612 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-11T16:25:14,612 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-11T16:25:14,615 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-12-11T16:25:14,615 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7446a65196a1,35865,1733934313218 in 271 msec
2024-12-11T16:25:14,618 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-12-11T16:25:14,619 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 507 msec
2024-12-11T16:25:14,620 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-11T16:25:14,620 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-12-11T16:25:14,621 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-11T16:25:14,621 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7446a65196a1,35865,1733934313218, seqNum=-1]
2024-12-11T16:25:14,622 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-11T16:25:14,627 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-13-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44525, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-11T16:25:14,645 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 993 msec
2024-12-11T16:25:14,645 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733934314645, completionTime=-1
2024-12-11T16:25:14,645 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-12-11T16:25:14,645 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-12-11T16:25:14,649 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-12-11T16:25:14,649 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733934374649
2024-12-11T16:25:14,649 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733934434649
2024-12-11T16:25:14,649 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 3 msec
2024-12-11T16:25:14,650 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7446a65196a1,35189,1733934312805-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-11T16:25:14,650 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7446a65196a1,35189,1733934312805-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T16:25:14,650 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7446a65196a1,35189,1733934312805-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T16:25:14,650 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-7446a65196a1:35189, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T16:25:14,650 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-11T16:25:14,650 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-12-11T16:25:14,653 DEBUG [master/7446a65196a1:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-12-11T16:25:14,656 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.296sec
2024-12-11T16:25:14,656 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-11T16:25:14,656 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-11T16:25:14,656 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-11T16:25:14,656 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-11T16:25:14,656 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-11T16:25:14,656 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7446a65196a1,35189,1733934312805-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-11T16:25:14,657 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7446a65196a1,35189,1733934312805-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-11T16:25:14,661 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-12-11T16:25:14,661 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-11T16:25:14,661 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7446a65196a1,35189,1733934312805-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T16:25:14,706 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36fd0d23, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-11T16:25:14,706 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7446a65196a1,35189,-1 for getting cluster id
2024-12-11T16:25:14,707 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-12-11T16:25:14,708 DEBUG [HMaster-EventLoopGroup-12-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7161a193-ea3b-4ed8-9e7d-95011ec824be'
2024-12-11T16:25:14,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-12-11T16:25:14,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7161a193-ea3b-4ed8-9e7d-95011ec824be"

2024-12-11T16:25:14,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50e61f09, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-11T16:25:14,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7446a65196a1,35189,-1]
2024-12-11T16:25:14,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-12-11T16:25:14,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T16:25:14,711 INFO  [HMaster-EventLoopGroup-12-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45008, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-12-11T16:25:14,712 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c4a5465, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-11T16:25:14,712 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-11T16:25:14,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7446a65196a1,35865,1733934313218, seqNum=-1]
2024-12-11T16:25:14,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-11T16:25:14,715 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53216, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-11T16:25:14,717 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=7446a65196a1,35189,1733934312805
2024-12-11T16:25:14,717 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T16:25:14,720 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-12-11T16:25:14,720 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-12-11T16:25:14,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncConnectionImpl(321): The fetched master address is 7446a65196a1,35189,1733934312805
2024-12-11T16:25:14,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@34c909b8
2024-12-11T16:25:14,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-12-11T16:25:14,722 INFO  [HMaster-EventLoopGroup-12-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45020, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-12-11T16:25:14,723 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35189 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-12-11T16:25:14,723 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35189 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-12-11T16:25:14,723 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35189 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testCompactionRecordDoesntBlockRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-11T16:25:14,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35189 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-11T16:25:14,726 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-12-11T16:25:14,726 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:25:14,726 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35189 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testCompactionRecordDoesntBlockRolling"
 procId is: 4
2024-12-11T16:25:14,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-11T16:25:14,728 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-12-11T16:25:14,744 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44201 is added to blk_1073741835_1011 (size=405)
2024-12-11T16:25:14,745 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44051 is added to blk_1073741835_1011 (size=405)
2024-12-11T16:25:14,748 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 078b25b941bc1defc22f6fbad96d961b, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733934314723.078b25b941bc1defc22f6fbad96d961b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testCompactionRecordDoesntBlockRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac
2024-12-11T16:25:14,764 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44051 is added to blk_1073741836_1012 (size=88)
2024-12-11T16:25:14,764 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44201 is added to blk_1073741836_1012 (size=88)
2024-12-11T16:25:14,765 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733934314723.078b25b941bc1defc22f6fbad96d961b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T16:25:14,766 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1722): Closing 078b25b941bc1defc22f6fbad96d961b, disabling compactions & flushes
2024-12-11T16:25:14,766 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733934314723.078b25b941bc1defc22f6fbad96d961b.
2024-12-11T16:25:14,766 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733934314723.078b25b941bc1defc22f6fbad96d961b.
2024-12-11T16:25:14,766 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733934314723.078b25b941bc1defc22f6fbad96d961b. after waiting 0 ms
2024-12-11T16:25:14,766 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733934314723.078b25b941bc1defc22f6fbad96d961b.
2024-12-11T16:25:14,766 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733934314723.078b25b941bc1defc22f6fbad96d961b.
2024-12-11T16:25:14,766 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for 078b25b941bc1defc22f6fbad96d961b:
Waiting for close lock at 1733934314766Disabling compacts and flushes for region at 1733934314766Disabling writes for close at 1733934314766Writing region close event to WAL at 1733934314766Closed at 1733934314766
2024-12-11T16:25:14,768 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ADD_TO_META
2024-12-11T16:25:14,768 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733934314723.078b25b941bc1defc22f6fbad96d961b.","families":{"info":[{"qualifier":"regioninfo","vlen":87,"tag":[],"timestamp":"1733934314768"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733934314768"}]},"ts":"1733934314768"}
2024-12-11T16:25:14,772 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-12-11T16:25:14,773 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-12-11T16:25:14,773 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733934314773"}]},"ts":"1733934314773"}
2024-12-11T16:25:14,776 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLING in hbase:meta
2024-12-11T16:25:14,777 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=078b25b941bc1defc22f6fbad96d961b, ASSIGN}]
2024-12-11T16:25:14,778 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=078b25b941bc1defc22f6fbad96d961b, ASSIGN
2024-12-11T16:25:14,779 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=078b25b941bc1defc22f6fbad96d961b, ASSIGN; state=OFFLINE, location=7446a65196a1,35865,1733934313218; forceNewPlan=false, retain=false
2024-12-11T16:25:14,930 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=078b25b941bc1defc22f6fbad96d961b, regionState=OPENING, regionLocation=7446a65196a1,35865,1733934313218
2024-12-11T16:25:14,933 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-13-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=078b25b941bc1defc22f6fbad96d961b, ASSIGN because future has completed
2024-12-11T16:25:14,944 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 078b25b941bc1defc22f6fbad96d961b, server=7446a65196a1,35865,1733934313218}]
2024-12-11T16:25:15,115 INFO  [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733934314723.078b25b941bc1defc22f6fbad96d961b.
2024-12-11T16:25:15,115 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 078b25b941bc1defc22f6fbad96d961b, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733934314723.078b25b941bc1defc22f6fbad96d961b.', STARTKEY => '', ENDKEY => ''}
2024-12-11T16:25:15,115 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testCompactionRecordDoesntBlockRolling 078b25b941bc1defc22f6fbad96d961b
2024-12-11T16:25:15,115 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733934314723.078b25b941bc1defc22f6fbad96d961b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T16:25:15,116 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 078b25b941bc1defc22f6fbad96d961b
2024-12-11T16:25:15,116 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 078b25b941bc1defc22f6fbad96d961b
2024-12-11T16:25:15,117 INFO  [StoreOpener-078b25b941bc1defc22f6fbad96d961b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 078b25b941bc1defc22f6fbad96d961b 
2024-12-11T16:25:15,119 INFO  [StoreOpener-078b25b941bc1defc22f6fbad96d961b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 078b25b941bc1defc22f6fbad96d961b columnFamilyName info
2024-12-11T16:25:15,119 DEBUG [StoreOpener-078b25b941bc1defc22f6fbad96d961b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:25:15,120 INFO  [StoreOpener-078b25b941bc1defc22f6fbad96d961b-1 {}] regionserver.HStore(327): Store=078b25b941bc1defc22f6fbad96d961b/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-11T16:25:15,120 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 078b25b941bc1defc22f6fbad96d961b
2024-12-11T16:25:15,122 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/078b25b941bc1defc22f6fbad96d961b
2024-12-11T16:25:15,123 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/078b25b941bc1defc22f6fbad96d961b
2024-12-11T16:25:15,124 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 078b25b941bc1defc22f6fbad96d961b
2024-12-11T16:25:15,124 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 078b25b941bc1defc22f6fbad96d961b
2024-12-11T16:25:15,126 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 078b25b941bc1defc22f6fbad96d961b
2024-12-11T16:25:15,131 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/078b25b941bc1defc22f6fbad96d961b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-11T16:25:15,133 INFO  [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 078b25b941bc1defc22f6fbad96d961b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=848460, jitterRate=0.07887372374534607}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-11T16:25:15,133 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 078b25b941bc1defc22f6fbad96d961b
2024-12-11T16:25:15,134 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 078b25b941bc1defc22f6fbad96d961b:
Running coprocessor pre-open hook at 1733934315116Writing region info on filesystem at 1733934315116Initializing all the Stores at 1733934315117 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733934315117Cleaning up temporary data from old regions at 1733934315124 (+7 ms)Running coprocessor post-open hooks at 1733934315133 (+9 ms)Region opened successfully at 1733934315134 (+1 ms)
2024-12-11T16:25:15,136 INFO  [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733934314723.078b25b941bc1defc22f6fbad96d961b., pid=6, masterSystemTime=1733934315101
2024-12-11T16:25:15,141 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733934314723.078b25b941bc1defc22f6fbad96d961b.
2024-12-11T16:25:15,141 INFO  [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733934314723.078b25b941bc1defc22f6fbad96d961b.
2024-12-11T16:25:15,146 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=078b25b941bc1defc22f6fbad96d961b, regionState=OPEN, openSeqNum=2, regionLocation=7446a65196a1,35865,1733934313218
2024-12-11T16:25:15,159 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35189 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=7446a65196a1,35865,1733934313218, table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=078b25b941bc1defc22f6fbad96d961b. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-12-11T16:25:15,165 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-13-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 078b25b941bc1defc22f6fbad96d961b, server=7446a65196a1,35865,1733934313218 because future has completed
2024-12-11T16:25:15,179 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-12-11T16:25:15,183 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 078b25b941bc1defc22f6fbad96d961b, server=7446a65196a1,35865,1733934313218 in 229 msec
2024-12-11T16:25:15,184 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-12-11T16:25:15,185 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=078b25b941bc1defc22f6fbad96d961b, ASSIGN in 403 msec
2024-12-11T16:25:15,187 INFO  [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-12-11T16:25:15,187 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733934315187"}]},"ts":"1733934315187"}
2024-12-11T16:25:15,195 INFO  [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLED in hbase:meta
2024-12-11T16:25:15,203 INFO  [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_POST_OPERATION
2024-12-11T16:25:15,210 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 480 msec
2024-12-11T16:25:15,338 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:15,348 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:16,339 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:16,349 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:17,192 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-12-11T16:25:17,192 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-12-11T16:25:17,193 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-11T16:25:17,193 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-12-11T16:25:17,193 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-12-11T16:25:17,193 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer
2024-12-11T16:25:17,193 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-11T16:25:17,194 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling Metrics about Tables on a single HBase RegionServer
2024-12-11T16:25:17,340 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:17,350 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:18,340 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:18,350 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:19,341 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:19,351 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:19,521 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:25:19,521 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:25:19,522 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:25:19,522 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:25:19,522 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:25:19,522 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:25:19,541 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:25:19,541 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:25:19,541 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:25:19,542 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:25:19,542 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:25:19,542 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:25:19,544 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:25:19,544 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:25:19,544 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:25:19,546 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:25:20,049 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-11T16:25:20,050 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:25:20,050 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:25:20,050 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:25:20,051 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:25:20,051 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:25:20,051 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:25:20,072 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:25:20,072 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:25:20,073 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:25:20,073 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:25:20,074 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:25:20,074 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:25:20,079 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:25:20,079 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:25:20,079 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:25:20,082 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:25:20,089 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-12-11T16:25:20,090 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testCompactionRecordDoesntBlockRolling'
2024-12-11T16:25:20,341 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:20,352 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:21,342 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:21,352 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:22,343 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:22,353 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:23,343 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:23,354 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:24,345 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:24,355 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:24,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-11T16:25:24,740 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-12-11T16:25:24,740 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testCompactionRecordDoesntBlockRolling,, stopping at row=TestLogRolling-testCompactionRecordDoesntBlockRolling ,, for max=2147483647 with caching=100
2024-12-11T16:25:24,743 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-11T16:25:24,743 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733934314723.078b25b941bc1defc22f6fbad96d961b.
2024-12-11T16:25:24,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testCompactionRecordDoesntBlockRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733934314723.078b25b941bc1defc22f6fbad96d961b., hostname=7446a65196a1,35865,1733934313218, seqNum=2]
2024-12-11T16:25:24,754 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35189 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-11T16:25:24,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35189 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-11T16:25:24,759 INFO  [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-12-11T16:25:24,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7
2024-12-11T16:25:24,761 INFO  [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-12-11T16:25:24,762 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-12-11T16:25:24,925 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35865 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8
2024-12-11T16:25:24,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7446a65196a1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733934314723.078b25b941bc1defc22f6fbad96d961b.
2024-12-11T16:25:24,926 INFO  [RS_FLUSH_OPERATIONS-regionserver/7446a65196a1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 078b25b941bc1defc22f6fbad96d961b 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-12-11T16:25:24,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7446a65196a1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/078b25b941bc1defc22f6fbad96d961b/.tmp/info/15b5c9a577ad40478293032b6f5bdc96 is 1080, key is row0001/info:/1733934324748/Put/seqid=0
2024-12-11T16:25:24,947 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44201 is added to blk_1073741837_1013 (size=6033)
2024-12-11T16:25:24,947 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44051 is added to blk_1073741837_1013 (size=6033)
2024-12-11T16:25:24,948 INFO  [RS_FLUSH_OPERATIONS-regionserver/7446a65196a1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/078b25b941bc1defc22f6fbad96d961b/.tmp/info/15b5c9a577ad40478293032b6f5bdc96
2024-12-11T16:25:24,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7446a65196a1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/078b25b941bc1defc22f6fbad96d961b/.tmp/info/15b5c9a577ad40478293032b6f5bdc96 as hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/078b25b941bc1defc22f6fbad96d961b/info/15b5c9a577ad40478293032b6f5bdc96
2024-12-11T16:25:24,960 INFO  [RS_FLUSH_OPERATIONS-regionserver/7446a65196a1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/078b25b941bc1defc22f6fbad96d961b/info/15b5c9a577ad40478293032b6f5bdc96, entries=1, sequenceid=5, filesize=5.9 K
2024-12-11T16:25:24,961 INFO  [RS_FLUSH_OPERATIONS-regionserver/7446a65196a1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 078b25b941bc1defc22f6fbad96d961b in 35ms, sequenceid=5, compaction requested=false
2024-12-11T16:25:24,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7446a65196a1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 078b25b941bc1defc22f6fbad96d961b:

2024-12-11T16:25:24,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7446a65196a1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733934314723.078b25b941bc1defc22f6fbad96d961b.
2024-12-11T16:25:24,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7446a65196a1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8
2024-12-11T16:25:24,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35189 {}] master.HMaster(4169): Remote procedure done, pid=8
2024-12-11T16:25:24,968 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7
2024-12-11T16:25:24,968 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 203 msec
2024-12-11T16:25:24,970 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 214 msec
2024-12-11T16:25:25,345 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:25,356 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:26,346 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:26,356 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:27,347 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:27,357 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:28,348 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:28,358 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:29,348 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:29,358 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:30,349 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:30,359 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:31,350 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:31,360 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:32,350 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:32,360 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:33,351 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:33,361 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:34,352 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:34,362 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:34,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7
2024-12-11T16:25:34,810 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-12-11T16:25:34,814 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35189 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-11T16:25:34,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35189 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-11T16:25:34,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9
2024-12-11T16:25:34,816 INFO  [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-12-11T16:25:34,817 INFO  [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-12-11T16:25:34,817 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-12-11T16:25:34,970 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35865 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10
2024-12-11T16:25:34,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7446a65196a1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733934314723.078b25b941bc1defc22f6fbad96d961b.
2024-12-11T16:25:34,972 INFO  [RS_FLUSH_OPERATIONS-regionserver/7446a65196a1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 078b25b941bc1defc22f6fbad96d961b 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-12-11T16:25:34,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7446a65196a1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/078b25b941bc1defc22f6fbad96d961b/.tmp/info/2463c1a6f86144fbb4142e9c5480fdf4 is 1080, key is row0002/info:/1733934334812/Put/seqid=0
2024-12-11T16:25:34,987 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44201 is added to blk_1073741838_1014 (size=6033)
2024-12-11T16:25:34,988 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44051 is added to blk_1073741838_1014 (size=6033)
2024-12-11T16:25:34,988 INFO  [RS_FLUSH_OPERATIONS-regionserver/7446a65196a1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/078b25b941bc1defc22f6fbad96d961b/.tmp/info/2463c1a6f86144fbb4142e9c5480fdf4
2024-12-11T16:25:34,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7446a65196a1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/078b25b941bc1defc22f6fbad96d961b/.tmp/info/2463c1a6f86144fbb4142e9c5480fdf4 as hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/078b25b941bc1defc22f6fbad96d961b/info/2463c1a6f86144fbb4142e9c5480fdf4
2024-12-11T16:25:35,002 INFO  [RS_FLUSH_OPERATIONS-regionserver/7446a65196a1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/078b25b941bc1defc22f6fbad96d961b/info/2463c1a6f86144fbb4142e9c5480fdf4, entries=1, sequenceid=9, filesize=5.9 K
2024-12-11T16:25:35,003 INFO  [RS_FLUSH_OPERATIONS-regionserver/7446a65196a1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 078b25b941bc1defc22f6fbad96d961b in 32ms, sequenceid=9, compaction requested=false
2024-12-11T16:25:35,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7446a65196a1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 078b25b941bc1defc22f6fbad96d961b:

2024-12-11T16:25:35,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7446a65196a1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733934314723.078b25b941bc1defc22f6fbad96d961b.
2024-12-11T16:25:35,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7446a65196a1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10
2024-12-11T16:25:35,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35189 {}] master.HMaster(4169): Remote procedure done, pid=10
2024-12-11T16:25:35,008 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9
2024-12-11T16:25:35,008 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 188 msec
2024-12-11T16:25:35,010 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 195 msec
2024-12-11T16:25:35,352 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:35,353 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064 after 68052ms
java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor205.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T16:25:35,363 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:35,363 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta after 68050ms
java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor205.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T16:25:36,354 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:36,364 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:37,354 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:37,365 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:38,355 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:38,365 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:39,356 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:39,366 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:40,356 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:40,367 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:41,357 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:41,367 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:42,358 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:42,368 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:42,761 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-11T16:25:43,359 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:43,369 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:44,359 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:44,370 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:44,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9
2024-12-11T16:25:44,841 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-12-11T16:25:44,844 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7446a65196a1%2C35865%2C1733934313218.1733934344843
2024-12-11T16:25:44,848 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:25:44,848 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:25:44,848 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:25:44,848 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:25:44,849 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:25:44,849 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/WALs/7446a65196a1,35865,1733934313218/7446a65196a1%2C35865%2C1733934313218.1733934313914 with entries=8, filesize=5.41 KB; new WAL /user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/WALs/7446a65196a1,35865,1733934313218/7446a65196a1%2C35865%2C1733934313218.1733934344843
2024-12-11T16:25:44,850 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36963:36963),(127.0.0.1/127.0.0.1:40679:40679)]
2024-12-11T16:25:44,850 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/WALs/7446a65196a1,35865,1733934313218/7446a65196a1%2C35865%2C1733934313218.1733934313914 is not closed yet, will try archiving it next time
2024-12-11T16:25:44,850 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44051 is added to blk_1073741833_1009 (size=5546)
2024-12-11T16:25:44,851 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35189 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-11T16:25:44,851 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44201 is added to blk_1073741833_1009 (size=5546)
2024-12-11T16:25:44,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35189 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-11T16:25:44,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11
2024-12-11T16:25:44,853 INFO  [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-12-11T16:25:44,854 INFO  [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-12-11T16:25:44,854 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-12-11T16:25:45,007 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35865 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12
2024-12-11T16:25:45,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7446a65196a1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733934314723.078b25b941bc1defc22f6fbad96d961b.
2024-12-11T16:25:45,008 INFO  [RS_FLUSH_OPERATIONS-regionserver/7446a65196a1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 078b25b941bc1defc22f6fbad96d961b 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-12-11T16:25:45,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7446a65196a1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/078b25b941bc1defc22f6fbad96d961b/.tmp/info/6dad0aba35f44319857363846f48a0fa is 1080, key is row0003/info:/1733934344842/Put/seqid=0
2024-12-11T16:25:45,017 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44201 is added to blk_1073741840_1016 (size=6033)
2024-12-11T16:25:45,017 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44051 is added to blk_1073741840_1016 (size=6033)
2024-12-11T16:25:45,018 INFO  [RS_FLUSH_OPERATIONS-regionserver/7446a65196a1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/078b25b941bc1defc22f6fbad96d961b/.tmp/info/6dad0aba35f44319857363846f48a0fa
2024-12-11T16:25:45,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7446a65196a1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/078b25b941bc1defc22f6fbad96d961b/.tmp/info/6dad0aba35f44319857363846f48a0fa as hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/078b25b941bc1defc22f6fbad96d961b/info/6dad0aba35f44319857363846f48a0fa
2024-12-11T16:25:45,032 INFO  [RS_FLUSH_OPERATIONS-regionserver/7446a65196a1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/078b25b941bc1defc22f6fbad96d961b/info/6dad0aba35f44319857363846f48a0fa, entries=1, sequenceid=13, filesize=5.9 K
2024-12-11T16:25:45,034 INFO  [RS_FLUSH_OPERATIONS-regionserver/7446a65196a1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 078b25b941bc1defc22f6fbad96d961b in 27ms, sequenceid=13, compaction requested=true
2024-12-11T16:25:45,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7446a65196a1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 078b25b941bc1defc22f6fbad96d961b:

2024-12-11T16:25:45,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7446a65196a1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733934314723.078b25b941bc1defc22f6fbad96d961b.
2024-12-11T16:25:45,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7446a65196a1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12
2024-12-11T16:25:45,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35189 {}] master.HMaster(4169): Remote procedure done, pid=12
2024-12-11T16:25:45,040 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11
2024-12-11T16:25:45,040 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 183 msec
2024-12-11T16:25:45,042 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 190 msec
2024-12-11T16:25:45,361 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:45,370 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:46,362 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:46,371 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:47,362 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:47,371 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:48,363 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:48,372 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:49,363 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:49,372 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:50,364 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:50,373 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:51,365 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:51,373 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:52,365 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:52,374 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:53,366 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:53,375 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:54,367 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:54,375 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:54,716 INFO  [master/7446a65196a1:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-12-11T16:25:54,716 INFO  [master/7446a65196a1:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-12-11T16:25:54,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11
2024-12-11T16:25:54,931 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-12-11T16:25:54,931 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-11T16:25:54,933 DEBUG [Time-limited test {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 18099 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-11T16:25:54,933 DEBUG [Time-limited test {}] regionserver.HStore(1541): 078b25b941bc1defc22f6fbad96d961b/info is initiating minor compaction (all files)
2024-12-11T16:25:54,933 INFO  [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-11T16:25:54,933 INFO  [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-11T16:25:54,934 INFO  [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 078b25b941bc1defc22f6fbad96d961b/info in TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733934314723.078b25b941bc1defc22f6fbad96d961b.
2024-12-11T16:25:54,934 INFO  [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/078b25b941bc1defc22f6fbad96d961b/info/15b5c9a577ad40478293032b6f5bdc96, hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/078b25b941bc1defc22f6fbad96d961b/info/2463c1a6f86144fbb4142e9c5480fdf4, hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/078b25b941bc1defc22f6fbad96d961b/info/6dad0aba35f44319857363846f48a0fa] into tmpdir=hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/078b25b941bc1defc22f6fbad96d961b/.tmp, totalSize=17.7 K
2024-12-11T16:25:54,935 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 15b5c9a577ad40478293032b6f5bdc96, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1733934324748
2024-12-11T16:25:54,935 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 2463c1a6f86144fbb4142e9c5480fdf4, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=9, earliestPutTs=1733934334812
2024-12-11T16:25:54,936 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 6dad0aba35f44319857363846f48a0fa, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733934344842
2024-12-11T16:25:54,950 INFO  [Time-limited test {}] throttle.PressureAwareThroughputController(145): 078b25b941bc1defc22f6fbad96d961b#info#compaction#45 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-11T16:25:54,950 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/078b25b941bc1defc22f6fbad96d961b/.tmp/info/002b5b056e8f4960b030f64a9016ad15 is 1080, key is row0001/info:/1733934324748/Put/seqid=0
2024-12-11T16:25:54,954 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44051 is added to blk_1073741841_1017 (size=8296)
2024-12-11T16:25:54,955 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44201 is added to blk_1073741841_1017 (size=8296)
2024-12-11T16:25:54,961 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/078b25b941bc1defc22f6fbad96d961b/.tmp/info/002b5b056e8f4960b030f64a9016ad15 as hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/078b25b941bc1defc22f6fbad96d961b/info/002b5b056e8f4960b030f64a9016ad15
2024-12-11T16:25:54,966 INFO  [Time-limited test {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 078b25b941bc1defc22f6fbad96d961b/info of 078b25b941bc1defc22f6fbad96d961b into 002b5b056e8f4960b030f64a9016ad15(size=8.1 K), total size for store is 8.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-11T16:25:54,967 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 078b25b941bc1defc22f6fbad96d961b:

2024-12-11T16:25:54,969 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7446a65196a1%2C35865%2C1733934313218.1733934354969
2024-12-11T16:25:54,974 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:25:54,974 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:25:54,974 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:25:54,975 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:25:54,975 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:25:54,975 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/WALs/7446a65196a1,35865,1733934313218/7446a65196a1%2C35865%2C1733934313218.1733934344843 with entries=4, filesize=2.45 KB; new WAL /user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/WALs/7446a65196a1,35865,1733934313218/7446a65196a1%2C35865%2C1733934313218.1733934354969
2024-12-11T16:25:54,976 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36963:36963),(127.0.0.1/127.0.0.1:40679:40679)]
2024-12-11T16:25:54,976 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/WALs/7446a65196a1,35865,1733934313218/7446a65196a1%2C35865%2C1733934313218.1733934344843 is not closed yet, will try archiving it next time
2024-12-11T16:25:54,976 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44201 is added to blk_1073741839_1015 (size=2520)
2024-12-11T16:25:54,976 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44051 is added to blk_1073741839_1015 (size=2520)
2024-12-11T16:25:54,977 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/WALs/7446a65196a1,35865,1733934313218/7446a65196a1%2C35865%2C1733934313218.1733934313914 to hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/oldWALs/7446a65196a1%2C35865%2C1733934313218.1733934313914
2024-12-11T16:25:54,977 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35189 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-11T16:25:54,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35189 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-11T16:25:54,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13
2024-12-11T16:25:54,980 INFO  [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-12-11T16:25:54,980 INFO  [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-12-11T16:25:54,980 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-12-11T16:25:55,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35865 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14
2024-12-11T16:25:55,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7446a65196a1:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733934314723.078b25b941bc1defc22f6fbad96d961b.
2024-12-11T16:25:55,133 INFO  [RS_FLUSH_OPERATIONS-regionserver/7446a65196a1:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 078b25b941bc1defc22f6fbad96d961b 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-12-11T16:25:55,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7446a65196a1:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/078b25b941bc1defc22f6fbad96d961b/.tmp/info/c2bffb3acd4b4c029a8a08d549af3098 is 1080, key is row0000/info:/1733934354968/Put/seqid=0
2024-12-11T16:25:55,142 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44201 is added to blk_1073741843_1019 (size=6033)
2024-12-11T16:25:55,142 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44051 is added to blk_1073741843_1019 (size=6033)
2024-12-11T16:25:55,142 INFO  [RS_FLUSH_OPERATIONS-regionserver/7446a65196a1:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/078b25b941bc1defc22f6fbad96d961b/.tmp/info/c2bffb3acd4b4c029a8a08d549af3098
2024-12-11T16:25:55,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7446a65196a1:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/078b25b941bc1defc22f6fbad96d961b/.tmp/info/c2bffb3acd4b4c029a8a08d549af3098 as hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/078b25b941bc1defc22f6fbad96d961b/info/c2bffb3acd4b4c029a8a08d549af3098
2024-12-11T16:25:55,154 INFO  [RS_FLUSH_OPERATIONS-regionserver/7446a65196a1:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/078b25b941bc1defc22f6fbad96d961b/info/c2bffb3acd4b4c029a8a08d549af3098, entries=1, sequenceid=18, filesize=5.9 K
2024-12-11T16:25:55,155 INFO  [RS_FLUSH_OPERATIONS-regionserver/7446a65196a1:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 078b25b941bc1defc22f6fbad96d961b in 22ms, sequenceid=18, compaction requested=false
2024-12-11T16:25:55,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7446a65196a1:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 078b25b941bc1defc22f6fbad96d961b:

2024-12-11T16:25:55,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7446a65196a1:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733934314723.078b25b941bc1defc22f6fbad96d961b.
2024-12-11T16:25:55,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7446a65196a1:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14
2024-12-11T16:25:55,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35189 {}] master.HMaster(4169): Remote procedure done, pid=14
2024-12-11T16:25:55,159 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13
2024-12-11T16:25:55,159 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 177 msec
2024-12-11T16:25:55,162 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 183 msec
2024-12-11T16:25:55,368 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:55,376 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:56,368 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:56,377 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:57,369 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:57,377 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:58,370 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:58,378 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:59,370 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:25:59,378 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:00,116 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 078b25b941bc1defc22f6fbad96d961b, had cached 0 bytes from a total of 14329
2024-12-11T16:26:00,371 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:00,379 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:01,371 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:01,379 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:02,372 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:02,380 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:03,373 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:03,381 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:04,374 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:04,381 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:05,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13
2024-12-11T16:26:05,032 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-12-11T16:26:05,038 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7446a65196a1%2C35865%2C1733934313218.1733934365037
2024-12-11T16:26:05,045 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:26:05,045 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:26:05,045 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:26:05,045 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:26:05,046 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:26:05,046 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/WALs/7446a65196a1,35865,1733934313218/7446a65196a1%2C35865%2C1733934313218.1733934354969 with entries=3, filesize=1.97 KB; new WAL /user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/WALs/7446a65196a1,35865,1733934313218/7446a65196a1%2C35865%2C1733934313218.1733934365037
2024-12-11T16:26:05,047 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44201 is added to blk_1073741842_1018 (size=2026)
2024-12-11T16:26:05,048 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44051 is added to blk_1073741842_1018 (size=2026)
2024-12-11T16:26:05,048 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36963:36963),(127.0.0.1/127.0.0.1:40679:40679)]
2024-12-11T16:26:05,048 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/WALs/7446a65196a1,35865,1733934313218/7446a65196a1%2C35865%2C1733934313218.1733934354969 is not closed yet, will try archiving it next time
2024-12-11T16:26:05,048 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-12-11T16:26:05,048 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/WALs/7446a65196a1,35865,1733934313218/7446a65196a1%2C35865%2C1733934313218.1733934344843 to hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/oldWALs/7446a65196a1%2C35865%2C1733934313218.1733934344843
2024-12-11T16:26:05,049 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-11T16:26:05,049 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-11T16:26:05,049 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T16:26:05,049 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T16:26:05,049 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-12-11T16:26:05,049 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-12-11T16:26:05,049 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=730692559, stopped=false
2024-12-11T16:26:05,049 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=7446a65196a1,35189,1733934312805
2024-12-11T16:26:05,056 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35865-0x10015bfabe30001, quorum=127.0.0.1:54728, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-11T16:26:05,056 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35865-0x10015bfabe30001, quorum=127.0.0.1:54728, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:26:05,056 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35189-0x10015bfabe30000, quorum=127.0.0.1:54728, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-11T16:26:05,057 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35189-0x10015bfabe30000, quorum=127.0.0.1:54728, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:26:05,057 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-11T16:26:05,057 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-11T16:26:05,057 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-11T16:26:05,057 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35865-0x10015bfabe30001, quorum=127.0.0.1:54728, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-11T16:26:05,057 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T16:26:05,057 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '7446a65196a1,35865,1733934313218' *****
2024-12-11T16:26:05,057 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-12-11T16:26:05,058 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35189-0x10015bfabe30000, quorum=127.0.0.1:54728, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-11T16:26:05,058 INFO  [RS:0;7446a65196a1:35865 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-11T16:26:05,058 INFO  [RS:0;7446a65196a1:35865 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-11T16:26:05,058 INFO  [RS:0;7446a65196a1:35865 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-11T16:26:05,058 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-12-11T16:26:05,058 INFO  [RS:0;7446a65196a1:35865 {}] regionserver.HRegionServer(3091): Received CLOSE for 078b25b941bc1defc22f6fbad96d961b
2024-12-11T16:26:05,059 INFO  [RS:0;7446a65196a1:35865 {}] regionserver.HRegionServer(959): stopping server 7446a65196a1,35865,1733934313218
2024-12-11T16:26:05,059 INFO  [RS:0;7446a65196a1:35865 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-11T16:26:05,059 INFO  [RS:0;7446a65196a1:35865 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;7446a65196a1:35865.
2024-12-11T16:26:05,059 DEBUG [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 078b25b941bc1defc22f6fbad96d961b, disabling compactions & flushes
2024-12-11T16:26:05,059 DEBUG [RS:0;7446a65196a1:35865 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-11T16:26:05,059 INFO  [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733934314723.078b25b941bc1defc22f6fbad96d961b.
2024-12-11T16:26:05,059 DEBUG [RS:0;7446a65196a1:35865 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T16:26:05,059 DEBUG [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733934314723.078b25b941bc1defc22f6fbad96d961b.
2024-12-11T16:26:05,059 DEBUG [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733934314723.078b25b941bc1defc22f6fbad96d961b. after waiting 0 ms
2024-12-11T16:26:05,059 DEBUG [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733934314723.078b25b941bc1defc22f6fbad96d961b.
2024-12-11T16:26:05,059 INFO  [RS:0;7446a65196a1:35865 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-11T16:26:05,059 INFO  [RS:0;7446a65196a1:35865 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-11T16:26:05,059 INFO  [RS:0;7446a65196a1:35865 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-11T16:26:05,059 INFO  [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 078b25b941bc1defc22f6fbad96d961b 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-12-11T16:26:05,059 INFO  [RS:0;7446a65196a1:35865 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-12-11T16:26:05,059 INFO  [RS:0;7446a65196a1:35865 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-12-11T16:26:05,060 DEBUG [RS:0;7446a65196a1:35865 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 078b25b941bc1defc22f6fbad96d961b=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733934314723.078b25b941bc1defc22f6fbad96d961b.}
2024-12-11T16:26:05,060 DEBUG [RS:0;7446a65196a1:35865 {}] regionserver.HRegionServer(1351): Waiting on 078b25b941bc1defc22f6fbad96d961b, 1588230740
2024-12-11T16:26:05,060 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-11T16:26:05,060 INFO  [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-11T16:26:05,060 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-11T16:26:05,060 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-11T16:26:05,060 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-11T16:26:05,060 INFO  [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.89 KB heapSize=3.91 KB
2024-12-11T16:26:05,064 DEBUG [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/078b25b941bc1defc22f6fbad96d961b/.tmp/info/251c7fee47aa4145bc052324ecd2e762 is 1080, key is row0001/info:/1733934365034/Put/seqid=0
2024-12-11T16:26:05,072 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44201 is added to blk_1073741845_1021 (size=6033)
2024-12-11T16:26:05,073 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44051 is added to blk_1073741845_1021 (size=6033)
2024-12-11T16:26:05,074 INFO  [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=22 (bloomFilter=true), to=hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/078b25b941bc1defc22f6fbad96d961b/.tmp/info/251c7fee47aa4145bc052324ecd2e762
2024-12-11T16:26:05,082 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/data/hbase/meta/1588230740/.tmp/info/2d47fb09dfa9455d9178da115ab54c25 is 227, key is TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733934314723.078b25b941bc1defc22f6fbad96d961b./info:regioninfo/1733934315146/Put/seqid=0
2024-12-11T16:26:05,085 DEBUG [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/078b25b941bc1defc22f6fbad96d961b/.tmp/info/251c7fee47aa4145bc052324ecd2e762 as hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/078b25b941bc1defc22f6fbad96d961b/info/251c7fee47aa4145bc052324ecd2e762
2024-12-11T16:26:05,087 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44051 is added to blk_1073741846_1022 (size=7308)
2024-12-11T16:26:05,088 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44201 is added to blk_1073741846_1022 (size=7308)
2024-12-11T16:26:05,092 INFO  [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/078b25b941bc1defc22f6fbad96d961b/info/251c7fee47aa4145bc052324ecd2e762, entries=1, sequenceid=22, filesize=5.9 K
2024-12-11T16:26:05,094 INFO  [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/data/hbase/meta/1588230740/.tmp/info/2d47fb09dfa9455d9178da115ab54c25
2024-12-11T16:26:05,094 INFO  [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 078b25b941bc1defc22f6fbad96d961b in 35ms, sequenceid=22, compaction requested=true
2024-12-11T16:26:05,097 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733934314723.078b25b941bc1defc22f6fbad96d961b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/078b25b941bc1defc22f6fbad96d961b/info/15b5c9a577ad40478293032b6f5bdc96, hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/078b25b941bc1defc22f6fbad96d961b/info/2463c1a6f86144fbb4142e9c5480fdf4, hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/078b25b941bc1defc22f6fbad96d961b/info/6dad0aba35f44319857363846f48a0fa] to archive
2024-12-11T16:26:05,098 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733934314723.078b25b941bc1defc22f6fbad96d961b.-1 {}] backup.HFileArchiver(363): Archiving compacted files.
2024-12-11T16:26:05,100 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/078b25b941bc1defc22f6fbad96d961b/info/15b5c9a577ad40478293032b6f5bdc96 to hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/078b25b941bc1defc22f6fbad96d961b/info/15b5c9a577ad40478293032b6f5bdc96
2024-12-11T16:26:05,100 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/078b25b941bc1defc22f6fbad96d961b/info/6dad0aba35f44319857363846f48a0fa to hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/078b25b941bc1defc22f6fbad96d961b/info/6dad0aba35f44319857363846f48a0fa
2024-12-11T16:26:05,100 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/078b25b941bc1defc22f6fbad96d961b/info/2463c1a6f86144fbb4142e9c5480fdf4 to hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/078b25b941bc1defc22f6fbad96d961b/info/2463c1a6f86144fbb4142e9c5480fdf4
2024-12-11T16:26:05,101 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733934314723.078b25b941bc1defc22f6fbad96d961b.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=7446a65196a1:35189 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 16 more
2024-12-11T16:26:05,101 WARN  [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733934314723.078b25b941bc1defc22f6fbad96d961b.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [15b5c9a577ad40478293032b6f5bdc96=6033, 2463c1a6f86144fbb4142e9c5480fdf4=6033, 6dad0aba35f44319857363846f48a0fa=6033]
2024-12-11T16:26:05,104 DEBUG [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/078b25b941bc1defc22f6fbad96d961b/recovered.edits/25.seqid, newMaxSeqId=25, maxSeqId=1
2024-12-11T16:26:05,105 INFO  [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733934314723.078b25b941bc1defc22f6fbad96d961b.
2024-12-11T16:26:05,105 DEBUG [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 078b25b941bc1defc22f6fbad96d961b:
Waiting for close lock at 1733934365059Running coprocessor pre-close hooks at 1733934365059Disabling compacts and flushes for region at 1733934365059Disabling writes for close at 1733934365059Obtaining lock to block concurrent updates at 1733934365059Preparing flush snapshotting stores in 078b25b941bc1defc22f6fbad96d961b at 1733934365059Finished memstore snapshotting TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733934314723.078b25b941bc1defc22f6fbad96d961b., syncing WAL and waiting on mvcc, flushsize=dataSize=1076, getHeapSize=1392, getOffHeapSize=0, getCellsCount=1 at 1733934365060 (+1 ms)Flushing stores of TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733934314723.078b25b941bc1defc22f6fbad96d961b. at 1733934365060Flushing 078b25b941bc1defc22f6fbad96d961b/info: creating writer at 1733934365060Flushing 078b25b941bc1defc22f6fbad96d961b/info: appending metadata at 1733934365063 (+3 ms)Flushing 078b25b941bc1defc22f6fbad96d961b/info: closing flushed file at 1733934365063Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@8a7857a: reopening flushed file at 1733934365084 (+21 ms)Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 078b25b941bc1defc22f6fbad96d961b in 35ms, sequenceid=22, compaction requested=true at 1733934365094 (+10 ms)Writing region close event to WAL at 1733934365101 (+7 ms)Running coprocessor post-close hooks at 1733934365105 (+4 ms)Closed at 1733934365105
2024-12-11T16:26:05,105 DEBUG [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733934314723.078b25b941bc1defc22f6fbad96d961b.
2024-12-11T16:26:05,112 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/data/hbase/meta/1588230740/.tmp/ns/8aeecb19fb8945fc8700228c5124b472 is 43, key is default/ns:d/1733934314628/Put/seqid=0
2024-12-11T16:26:05,116 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44201 is added to blk_1073741847_1023 (size=5153)
2024-12-11T16:26:05,117 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44051 is added to blk_1073741847_1023 (size=5153)
2024-12-11T16:26:05,117 INFO  [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/data/hbase/meta/1588230740/.tmp/ns/8aeecb19fb8945fc8700228c5124b472
2024-12-11T16:26:05,134 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/data/hbase/meta/1588230740/.tmp/table/8ddb28726161403282c6901f86900f2e is 89, key is TestLogRolling-testCompactionRecordDoesntBlockRolling/table:state/1733934315187/Put/seqid=0
2024-12-11T16:26:05,138 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44201 is added to blk_1073741848_1024 (size=5508)
2024-12-11T16:26:05,138 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44051 is added to blk_1073741848_1024 (size=5508)
2024-12-11T16:26:05,138 INFO  [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=170 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/data/hbase/meta/1588230740/.tmp/table/8ddb28726161403282c6901f86900f2e
2024-12-11T16:26:05,144 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/data/hbase/meta/1588230740/.tmp/info/2d47fb09dfa9455d9178da115ab54c25 as hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/data/hbase/meta/1588230740/info/2d47fb09dfa9455d9178da115ab54c25
2024-12-11T16:26:05,149 INFO  [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/data/hbase/meta/1588230740/info/2d47fb09dfa9455d9178da115ab54c25, entries=10, sequenceid=11, filesize=7.1 K
2024-12-11T16:26:05,150 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/data/hbase/meta/1588230740/.tmp/ns/8aeecb19fb8945fc8700228c5124b472 as hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/data/hbase/meta/1588230740/ns/8aeecb19fb8945fc8700228c5124b472
2024-12-11T16:26:05,157 INFO  [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/data/hbase/meta/1588230740/ns/8aeecb19fb8945fc8700228c5124b472, entries=2, sequenceid=11, filesize=5.0 K
2024-12-11T16:26:05,158 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/data/hbase/meta/1588230740/.tmp/table/8ddb28726161403282c6901f86900f2e as hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/data/hbase/meta/1588230740/table/8ddb28726161403282c6901f86900f2e
2024-12-11T16:26:05,165 INFO  [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/data/hbase/meta/1588230740/table/8ddb28726161403282c6901f86900f2e, entries=2, sequenceid=11, filesize=5.4 K
2024-12-11T16:26:05,166 INFO  [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.89 KB/1932, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1588230740 in 106ms, sequenceid=11, compaction requested=false
2024-12-11T16:26:05,170 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-12-11T16:26:05,171 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-11T16:26:05,171 INFO  [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-11T16:26:05,171 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733934365060Running coprocessor pre-close hooks at 1733934365060Disabling compacts and flushes for region at 1733934365060Disabling writes for close at 1733934365060Obtaining lock to block concurrent updates at 1733934365060Preparing flush snapshotting stores in 1588230740 at 1733934365060Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1932, getHeapSize=3936, getOffHeapSize=0, getCellsCount=14 at 1733934365060Flushing stores of hbase:meta,,1.1588230740 at 1733934365061 (+1 ms)Flushing 1588230740/info: creating writer at 1733934365061Flushing 1588230740/info: appending metadata at 1733934365082 (+21 ms)Flushing 1588230740/info: closing flushed file at 1733934365082Flushing 1588230740/ns: creating writer at 1733934365100 (+18 ms)Flushing 1588230740/ns: appending metadata at 1733934365112 (+12 ms)Flushing 1588230740/ns: closing flushed file at 1733934365112Flushing 1588230740/table: creating writer at 1733934365121 (+9 ms)Flushing 1588230740/table: appending metadata at 1733934365133 (+12 ms)Flushing 1588230740/table: closing flushed file at 1733934365133Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@62ba08d5: reopening flushed file at 1733934365143 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@10f31b9c: reopening flushed file at 1733934365149 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3efbfda5: reopening flushed file at 1733934365158 (+9 ms)Finished flush of dataSize ~1.89 KB/1932, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1588230740 in 106ms, sequenceid=11, compaction requested=false at 1733934365166 (+8 ms)Writing region close event to WAL at 1733934365167 (+1 ms)Running coprocessor post-close hooks at 1733934365171 (+4 ms)Closed at 1733934365171
2024-12-11T16:26:05,171 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-12-11T16:26:05,260 INFO  [RS:0;7446a65196a1:35865 {}] regionserver.HRegionServer(976): stopping server 7446a65196a1,35865,1733934313218; all regions closed.
2024-12-11T16:26:05,260 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:26:05,260 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:26:05,260 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:26:05,261 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:26:05,261 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:26:05,263 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44201 is added to blk_1073741834_1010 (size=3306)
2024-12-11T16:26:05,263 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44051 is added to blk_1073741834_1010 (size=3306)
2024-12-11T16:26:05,375 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:05,382 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:05,668 DEBUG [RS:0;7446a65196a1:35865 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/oldWALs
2024-12-11T16:26:05,668 INFO  [RS:0;7446a65196a1:35865 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 7446a65196a1%2C35865%2C1733934313218.meta:.meta(num 1733934314503)
2024-12-11T16:26:05,669 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:26:05,669 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:26:05,669 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:26:05,670 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:26:05,670 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:26:05,673 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44201 is added to blk_1073741844_1020 (size=1252)
2024-12-11T16:26:05,674 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44051 is added to blk_1073741844_1020 (size=1252)
2024-12-11T16:26:05,678 DEBUG [RS:0;7446a65196a1:35865 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/oldWALs
2024-12-11T16:26:05,678 INFO  [RS:0;7446a65196a1:35865 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 7446a65196a1%2C35865%2C1733934313218:(num 1733934365037)
2024-12-11T16:26:05,678 DEBUG [RS:0;7446a65196a1:35865 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T16:26:05,678 INFO  [RS:0;7446a65196a1:35865 {}] regionserver.LeaseManager(133): Closed leases
2024-12-11T16:26:05,678 INFO  [RS:0;7446a65196a1:35865 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-11T16:26:05,678 INFO  [RS:0;7446a65196a1:35865 {}] hbase.ChoreService(370): Chore service for: regionserver/7446a65196a1:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown
2024-12-11T16:26:05,679 INFO  [RS:0;7446a65196a1:35865 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-11T16:26:05,679 INFO  [regionserver/7446a65196a1:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-11T16:26:05,679 INFO  [RS:0;7446a65196a1:35865 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35865
2024-12-11T16:26:05,731 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35865-0x10015bfabe30001, quorum=127.0.0.1:54728, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/7446a65196a1,35865,1733934313218
2024-12-11T16:26:05,731 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35189-0x10015bfabe30000, quorum=127.0.0.1:54728, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-11T16:26:05,731 INFO  [RS:0;7446a65196a1:35865 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-11T16:26:05,732 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [7446a65196a1,35865,1733934313218]
2024-12-11T16:26:05,748 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/7446a65196a1,35865,1733934313218 already deleted, retry=false
2024-12-11T16:26:05,748 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 7446a65196a1,35865,1733934313218 expired; onlineServers=0
2024-12-11T16:26:05,748 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '7446a65196a1,35189,1733934312805' *****
2024-12-11T16:26:05,748 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-12-11T16:26:05,748 INFO  [M:0;7446a65196a1:35189 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-11T16:26:05,748 INFO  [M:0;7446a65196a1:35189 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-11T16:26:05,748 DEBUG [M:0;7446a65196a1:35189 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-12-11T16:26:05,748 DEBUG [M:0;7446a65196a1:35189 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-12-11T16:26:05,748 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-12-11T16:26:05,748 DEBUG [master/7446a65196a1:0:becomeActiveMaster-HFileCleaner.large.0-1733934313653 {}] cleaner.HFileCleaner(306): Exit Thread[master/7446a65196a1:0:becomeActiveMaster-HFileCleaner.large.0-1733934313653,5,FailOnTimeoutGroup]
2024-12-11T16:26:05,748 DEBUG [master/7446a65196a1:0:becomeActiveMaster-HFileCleaner.small.0-1733934313653 {}] cleaner.HFileCleaner(306): Exit Thread[master/7446a65196a1:0:becomeActiveMaster-HFileCleaner.small.0-1733934313653,5,FailOnTimeoutGroup]
2024-12-11T16:26:05,748 INFO  [M:0;7446a65196a1:35189 {}] hbase.ChoreService(370): Chore service for: master/7446a65196a1:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-12-11T16:26:05,749 INFO  [M:0;7446a65196a1:35189 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-11T16:26:05,749 DEBUG [M:0;7446a65196a1:35189 {}] master.HMaster(1795): Stopping service threads
2024-12-11T16:26:05,749 INFO  [M:0;7446a65196a1:35189 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-12-11T16:26:05,749 INFO  [M:0;7446a65196a1:35189 {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-11T16:26:05,749 INFO  [M:0;7446a65196a1:35189 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-12-11T16:26:05,749 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-12-11T16:26:05,756 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35189-0x10015bfabe30000, quorum=127.0.0.1:54728, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-12-11T16:26:05,756 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35189-0x10015bfabe30000, quorum=127.0.0.1:54728, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:26:05,756 DEBUG [M:0;7446a65196a1:35189 {}] zookeeper.ZKUtil(347): master:35189-0x10015bfabe30000, quorum=127.0.0.1:54728, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-12-11T16:26:05,756 WARN  [M:0;7446a65196a1:35189 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-12-11T16:26:05,757 INFO  [M:0;7446a65196a1:35189 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/.lastflushedseqids
2024-12-11T16:26:05,763 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44201 is added to blk_1073741849_1025 (size=130)
2024-12-11T16:26:05,763 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44051 is added to blk_1073741849_1025 (size=130)
2024-12-11T16:26:05,764 INFO  [M:0;7446a65196a1:35189 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-12-11T16:26:05,764 INFO  [M:0;7446a65196a1:35189 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-12-11T16:26:05,764 DEBUG [M:0;7446a65196a1:35189 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-11T16:26:05,764 INFO  [M:0;7446a65196a1:35189 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T16:26:05,764 DEBUG [M:0;7446a65196a1:35189 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T16:26:05,764 DEBUG [M:0;7446a65196a1:35189 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-11T16:26:05,764 DEBUG [M:0;7446a65196a1:35189 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T16:26:05,764 INFO  [M:0;7446a65196a1:35189 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=43.60 KB heapSize=55.01 KB
2024-12-11T16:26:05,779 INFO  [regionserver/7446a65196a1:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-11T16:26:05,783 DEBUG [M:0;7446a65196a1:35189 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d54d513d27bb4efaa352944e9b8b1f6a is 82, key is hbase:meta,,1/info:regioninfo/1733934314536/Put/seqid=0
2024-12-11T16:26:05,787 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44051 is added to blk_1073741850_1026 (size=5672)
2024-12-11T16:26:05,788 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44201 is added to blk_1073741850_1026 (size=5672)
2024-12-11T16:26:05,788 INFO  [M:0;7446a65196a1:35189 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d54d513d27bb4efaa352944e9b8b1f6a
2024-12-11T16:26:05,806 DEBUG [M:0;7446a65196a1:35189 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/899c3019a96e4241902ca9ad6b12369f is 799, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733934315209/Put/seqid=0
2024-12-11T16:26:05,810 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44051 is added to blk_1073741851_1027 (size=7824)
2024-12-11T16:26:05,810 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44201 is added to blk_1073741851_1027 (size=7824)
2024-12-11T16:26:05,811 INFO  [M:0;7446a65196a1:35189 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=43.00 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/899c3019a96e4241902ca9ad6b12369f
2024-12-11T16:26:05,815 INFO  [M:0;7446a65196a1:35189 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 899c3019a96e4241902ca9ad6b12369f
2024-12-11T16:26:05,829 DEBUG [M:0;7446a65196a1:35189 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9cb82387a0644e3b8714aedfe0bbf1b7 is 69, key is 7446a65196a1,35865,1733934313218/rs:state/1733934313752/Put/seqid=0
2024-12-11T16:26:05,833 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44201 is added to blk_1073741852_1028 (size=5156)
2024-12-11T16:26:05,833 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44051 is added to blk_1073741852_1028 (size=5156)
2024-12-11T16:26:05,834 INFO  [M:0;7446a65196a1:35189 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9cb82387a0644e3b8714aedfe0bbf1b7
2024-12-11T16:26:05,840 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35865-0x10015bfabe30001, quorum=127.0.0.1:54728, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-11T16:26:05,840 INFO  [RS:0;7446a65196a1:35865 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-11T16:26:05,840 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35865-0x10015bfabe30001, quorum=127.0.0.1:54728, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-11T16:26:05,840 INFO  [RS:0;7446a65196a1:35865 {}] regionserver.HRegionServer(1031): Exiting; stopping=7446a65196a1,35865,1733934313218; zookeeper connection closed.
2024-12-11T16:26:05,840 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3830d0a3 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3830d0a3
2024-12-11T16:26:05,840 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-12-11T16:26:05,850 DEBUG [M:0;7446a65196a1:35189 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/4d23bf16980046f0af62fbe6b125cd80 is 52, key is load_balancer_on/state:d/1733934314718/Put/seqid=0
2024-12-11T16:26:05,854 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44201 is added to blk_1073741853_1029 (size=5056)
2024-12-11T16:26:05,854 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44051 is added to blk_1073741853_1029 (size=5056)
2024-12-11T16:26:05,855 INFO  [M:0;7446a65196a1:35189 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/4d23bf16980046f0af62fbe6b125cd80
2024-12-11T16:26:05,859 DEBUG [M:0;7446a65196a1:35189 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d54d513d27bb4efaa352944e9b8b1f6a as hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/d54d513d27bb4efaa352944e9b8b1f6a
2024-12-11T16:26:05,863 INFO  [M:0;7446a65196a1:35189 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/d54d513d27bb4efaa352944e9b8b1f6a, entries=8, sequenceid=121, filesize=5.5 K
2024-12-11T16:26:05,864 DEBUG [M:0;7446a65196a1:35189 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/899c3019a96e4241902ca9ad6b12369f as hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/899c3019a96e4241902ca9ad6b12369f
2024-12-11T16:26:05,869 INFO  [M:0;7446a65196a1:35189 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 899c3019a96e4241902ca9ad6b12369f
2024-12-11T16:26:05,869 INFO  [M:0;7446a65196a1:35189 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/899c3019a96e4241902ca9ad6b12369f, entries=14, sequenceid=121, filesize=7.6 K
2024-12-11T16:26:05,870 DEBUG [M:0;7446a65196a1:35189 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9cb82387a0644e3b8714aedfe0bbf1b7 as hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/9cb82387a0644e3b8714aedfe0bbf1b7
2024-12-11T16:26:05,874 INFO  [M:0;7446a65196a1:35189 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/9cb82387a0644e3b8714aedfe0bbf1b7, entries=1, sequenceid=121, filesize=5.0 K
2024-12-11T16:26:05,875 DEBUG [M:0;7446a65196a1:35189 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/4d23bf16980046f0af62fbe6b125cd80 as hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/4d23bf16980046f0af62fbe6b125cd80
2024-12-11T16:26:05,880 INFO  [M:0;7446a65196a1:35189 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43205/user/jenkins/test-data/67c3cef4-8efa-64b9-6068-00fb999365ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/4d23bf16980046f0af62fbe6b125cd80, entries=1, sequenceid=121, filesize=4.9 K
2024-12-11T16:26:05,881 INFO  [M:0;7446a65196a1:35189 {}] regionserver.HRegion(3140): Finished flush of dataSize ~43.60 KB/44650, heapSize ~54.95 KB/56264, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 117ms, sequenceid=121, compaction requested=false
2024-12-11T16:26:05,883 INFO  [M:0;7446a65196a1:35189 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T16:26:05,883 DEBUG [M:0;7446a65196a1:35189 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733934365764Disabling compacts and flushes for region at 1733934365764Disabling writes for close at 1733934365764Obtaining lock to block concurrent updates at 1733934365764Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733934365764Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=44650, getHeapSize=56264, getOffHeapSize=0, getCellsCount=140 at 1733934365765 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733934365765Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733934365766 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733934365783 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733934365783Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733934365792 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733934365805 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733934365805Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733934365815 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733934365828 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733934365828Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733934365838 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733934365850 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733934365850Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@65784df4: reopening flushed file at 1733934365858 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@560fc60d: reopening flushed file at 1733934365863 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@67715e20: reopening flushed file at 1733934365869 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6d453259: reopening flushed file at 1733934365874 (+5 ms)Finished flush of dataSize ~43.60 KB/44650, heapSize ~54.95 KB/56264, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 117ms, sequenceid=121, compaction requested=false at 1733934365881 (+7 ms)Writing region close event to WAL at 1733934365883 (+2 ms)Closed at 1733934365883
2024-12-11T16:26:05,883 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:26:05,883 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:26:05,883 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:26:05,883 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:26:05,883 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:26:05,885 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44051 is added to blk_1073741830_1006 (size=53047)
2024-12-11T16:26:05,885 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44201 is added to blk_1073741830_1006 (size=53047)
2024-12-11T16:26:05,886 INFO  [M:0;7446a65196a1:35189 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-12-11T16:26:05,886 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-11T16:26:05,886 INFO  [M:0;7446a65196a1:35189 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35189
2024-12-11T16:26:05,886 INFO  [M:0;7446a65196a1:35189 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-11T16:26:06,015 INFO  [M:0;7446a65196a1:35189 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-11T16:26:06,015 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35189-0x10015bfabe30000, quorum=127.0.0.1:54728, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-11T16:26:06,015 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35189-0x10015bfabe30000, quorum=127.0.0.1:54728, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-11T16:26:06,017 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5f44ac55{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T16:26:06,017 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@303cb3d3{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-11T16:26:06,017 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-11T16:26:06,017 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@26ab0093{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-11T16:26:06,017 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6bc5fff1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffdf3259-260d-ac79-a7e6-520e991bd2d0/hadoop.log.dir/,STOPPED}
2024-12-11T16:26:06,019 WARN  [BP-1668126573-172.17.0.2-1733934310453 heartbeating to localhost/127.0.0.1:43205 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-11T16:26:06,019 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-11T16:26:06,019 WARN  [BP-1668126573-172.17.0.2-1733934310453 heartbeating to localhost/127.0.0.1:43205 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1668126573-172.17.0.2-1733934310453 (Datanode Uuid 3aee8323-9ec0-4965-ac7c-a730bbf84d01) service to localhost/127.0.0.1:43205
2024-12-11T16:26:06,019 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-11T16:26:06,019 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffdf3259-260d-ac79-a7e6-520e991bd2d0/cluster_9179545b-40d5-c31c-24dd-6aa8b56461ff/data/data3/current/BP-1668126573-172.17.0.2-1733934310453 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T16:26:06,019 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffdf3259-260d-ac79-a7e6-520e991bd2d0/cluster_9179545b-40d5-c31c-24dd-6aa8b56461ff/data/data4/current/BP-1668126573-172.17.0.2-1733934310453 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T16:26:06,020 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-11T16:26:06,023 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@17065767{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T16:26:06,024 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@374344fc{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-11T16:26:06,024 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-11T16:26:06,024 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5785ef07{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-11T16:26:06,024 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@82bb36f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffdf3259-260d-ac79-a7e6-520e991bd2d0/hadoop.log.dir/,STOPPED}
2024-12-11T16:26:06,026 WARN  [BP-1668126573-172.17.0.2-1733934310453 heartbeating to localhost/127.0.0.1:43205 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-11T16:26:06,026 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-11T16:26:06,026 WARN  [BP-1668126573-172.17.0.2-1733934310453 heartbeating to localhost/127.0.0.1:43205 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1668126573-172.17.0.2-1733934310453 (Datanode Uuid 784914ec-be2b-45eb-98b4-d953662031e6) service to localhost/127.0.0.1:43205
2024-12-11T16:26:06,026 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-11T16:26:06,026 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffdf3259-260d-ac79-a7e6-520e991bd2d0/cluster_9179545b-40d5-c31c-24dd-6aa8b56461ff/data/data1/current/BP-1668126573-172.17.0.2-1733934310453 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T16:26:06,026 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffdf3259-260d-ac79-a7e6-520e991bd2d0/cluster_9179545b-40d5-c31c-24dd-6aa8b56461ff/data/data2/current/BP-1668126573-172.17.0.2-1733934310453 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T16:26:06,026 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-11T16:26:06,031 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6f8ea346{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-11T16:26:06,032 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@59ed7094{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-11T16:26:06,032 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-11T16:26:06,032 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1cf94809{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-11T16:26:06,032 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4182d22c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffdf3259-260d-ac79-a7e6-520e991bd2d0/hadoop.log.dir/,STOPPED}
2024-12-11T16:26:06,037 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-12-11T16:26:06,055 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-12-11T16:26:06,062 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testCompactionRecordDoesntBlockRolling Thread=206 (was 179)
Potentially hanging thread: HMaster-EventLoopGroup-12-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-35-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-36-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43205
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-12
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-34-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-35-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43205
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:43205
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-37-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (548703531) connection to localhost/127.0.0.1:43205 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: IPC Client (548703531) connection to localhost/127.0.0.1:43205 from jenkins.hfs.5
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43205
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-36-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.5@localhost:43205
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-35-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-34-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (548703531) connection to localhost/127.0.0.1:43205 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-36-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-37-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-12-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-10
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-34-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-12-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-11
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-37-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=485 (was 455) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=417 (was 627), ProcessCount=11 (was 11), AvailableMemoryMB=252 (was 1134)
2024-12-11T16:26:06,068 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRolling Thread=206, OpenFileDescriptor=485, MaxFileDescriptor=1048576, SystemLoadAverage=417, ProcessCount=11, AvailableMemoryMB=252
2024-12-11T16:26:06,069 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-12-11T16:26:06,069 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffdf3259-260d-ac79-a7e6-520e991bd2d0/hadoop.log.dir so I do NOT create it in target/test-data/91bb4f8c-44ab-4329-ecbc-8f1e37d48355
2024-12-11T16:26:06,069 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffdf3259-260d-ac79-a7e6-520e991bd2d0/hadoop.tmp.dir so I do NOT create it in target/test-data/91bb4f8c-44ab-4329-ecbc-8f1e37d48355
2024-12-11T16:26:06,069 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91bb4f8c-44ab-4329-ecbc-8f1e37d48355/cluster_8a707b7d-e55b-61a4-30e3-3320dbb3250b, deleteOnExit=true
2024-12-11T16:26:06,069 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-12-11T16:26:06,069 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91bb4f8c-44ab-4329-ecbc-8f1e37d48355/test.cache.data in system properties and HBase conf
2024-12-11T16:26:06,069 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91bb4f8c-44ab-4329-ecbc-8f1e37d48355/hadoop.tmp.dir in system properties and HBase conf
2024-12-11T16:26:06,069 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91bb4f8c-44ab-4329-ecbc-8f1e37d48355/hadoop.log.dir in system properties and HBase conf
2024-12-11T16:26:06,069 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91bb4f8c-44ab-4329-ecbc-8f1e37d48355/mapreduce.cluster.local.dir in system properties and HBase conf
2024-12-11T16:26:06,069 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91bb4f8c-44ab-4329-ecbc-8f1e37d48355/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-12-11T16:26:06,070 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-12-11T16:26:06,070 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-11T16:26:06,070 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91bb4f8c-44ab-4329-ecbc-8f1e37d48355/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-12-11T16:26:06,070 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91bb4f8c-44ab-4329-ecbc-8f1e37d48355/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-12-11T16:26:06,070 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91bb4f8c-44ab-4329-ecbc-8f1e37d48355/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-12-11T16:26:06,070 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91bb4f8c-44ab-4329-ecbc-8f1e37d48355/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-11T16:26:06,070 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91bb4f8c-44ab-4329-ecbc-8f1e37d48355/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-12-11T16:26:06,070 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91bb4f8c-44ab-4329-ecbc-8f1e37d48355/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-12-11T16:26:06,070 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91bb4f8c-44ab-4329-ecbc-8f1e37d48355/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-11T16:26:06,070 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91bb4f8c-44ab-4329-ecbc-8f1e37d48355/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-11T16:26:06,070 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91bb4f8c-44ab-4329-ecbc-8f1e37d48355/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-12-11T16:26:06,071 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91bb4f8c-44ab-4329-ecbc-8f1e37d48355/nfs.dump.dir in system properties and HBase conf
2024-12-11T16:26:06,071 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91bb4f8c-44ab-4329-ecbc-8f1e37d48355/java.io.tmpdir in system properties and HBase conf
2024-12-11T16:26:06,071 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91bb4f8c-44ab-4329-ecbc-8f1e37d48355/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-11T16:26:06,071 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91bb4f8c-44ab-4329-ecbc-8f1e37d48355/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-12-11T16:26:06,071 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91bb4f8c-44ab-4329-ecbc-8f1e37d48355/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-12-11T16:26:06,083 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-11T16:26:06,348 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-11T16:26:06,352 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-11T16:26:06,354 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-11T16:26:06,354 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-11T16:26:06,354 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-11T16:26:06,354 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-11T16:26:06,355 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@48c6491c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91bb4f8c-44ab-4329-ecbc-8f1e37d48355/hadoop.log.dir/,AVAILABLE}
2024-12-11T16:26:06,355 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7abfc7d2{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-11T16:26:06,375 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:06,383 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:06,467 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@49453556{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91bb4f8c-44ab-4329-ecbc-8f1e37d48355/java.io.tmpdir/jetty-localhost-37041-hadoop-hdfs-3_4_1-tests_jar-_-any-15148952892194315084/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-11T16:26:06,467 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2bc7bf21{HTTP/1.1, (http/1.1)}{localhost:37041}
2024-12-11T16:26:06,467 INFO  [Time-limited test {}] server.Server(415): Started @257138ms
2024-12-11T16:26:06,483 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-11T16:26:06,717 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-11T16:26:06,720 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-11T16:26:06,721 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-11T16:26:06,721 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-11T16:26:06,721 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-11T16:26:06,721 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2cf5c358{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91bb4f8c-44ab-4329-ecbc-8f1e37d48355/hadoop.log.dir/,AVAILABLE}
2024-12-11T16:26:06,722 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@57d7e59e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-11T16:26:06,842 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@58d3c043{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91bb4f8c-44ab-4329-ecbc-8f1e37d48355/java.io.tmpdir/jetty-localhost-33243-hadoop-hdfs-3_4_1-tests_jar-_-any-5626429629838037039/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T16:26:06,843 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7b1dc3c7{HTTP/1.1, (http/1.1)}{localhost:33243}
2024-12-11T16:26:06,843 INFO  [Time-limited test {}] server.Server(415): Started @257513ms
2024-12-11T16:26:06,844 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-11T16:26:06,879 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-11T16:26:06,882 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-11T16:26:06,883 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-11T16:26:06,883 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-11T16:26:06,883 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-11T16:26:06,883 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4f9d87e6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91bb4f8c-44ab-4329-ecbc-8f1e37d48355/hadoop.log.dir/,AVAILABLE}
2024-12-11T16:26:06,884 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@24fba7ba{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-11T16:26:06,975 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@24e33e74{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91bb4f8c-44ab-4329-ecbc-8f1e37d48355/java.io.tmpdir/jetty-localhost-41769-hadoop-hdfs-3_4_1-tests_jar-_-any-3180124013126126956/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T16:26:06,975 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3e392bcf{HTTP/1.1, (http/1.1)}{localhost:41769}
2024-12-11T16:26:06,975 INFO  [Time-limited test {}] server.Server(415): Started @257645ms
2024-12-11T16:26:06,976 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-11T16:26:07,192 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-11T16:26:07,192 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-12-11T16:26:07,192 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-12-11T16:26:07,193 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-11T16:26:07,376 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:07,383 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:07,529 WARN  [Thread-1972 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91bb4f8c-44ab-4329-ecbc-8f1e37d48355/cluster_8a707b7d-e55b-61a4-30e3-3320dbb3250b/data/data1/current/BP-470135049-172.17.0.2-1733934366087/current, will proceed with Du for space computation calculation, 
2024-12-11T16:26:07,529 WARN  [Thread-1973 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91bb4f8c-44ab-4329-ecbc-8f1e37d48355/cluster_8a707b7d-e55b-61a4-30e3-3320dbb3250b/data/data2/current/BP-470135049-172.17.0.2-1733934366087/current, will proceed with Du for space computation calculation, 
2024-12-11T16:26:07,548 WARN  [Thread-1936 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-11T16:26:07,550 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x57ae998ae5598f13 with lease ID 0x716ea05754cf13a4: Processing first storage report for DS-a4b9f19c-6e1e-46d0-ab5d-24b84f1429b8 from datanode DatanodeRegistration(127.0.0.1:45613, datanodeUuid=9a84d457-85df-4ef8-9704-f07ec07f9ac5, infoPort=34135, infoSecurePort=0, ipcPort=34237, storageInfo=lv=-57;cid=testClusterID;nsid=1506381876;c=1733934366087)
2024-12-11T16:26:07,550 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x57ae998ae5598f13 with lease ID 0x716ea05754cf13a4: from storage DS-a4b9f19c-6e1e-46d0-ab5d-24b84f1429b8 node DatanodeRegistration(127.0.0.1:45613, datanodeUuid=9a84d457-85df-4ef8-9704-f07ec07f9ac5, infoPort=34135, infoSecurePort=0, ipcPort=34237, storageInfo=lv=-57;cid=testClusterID;nsid=1506381876;c=1733934366087), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-11T16:26:07,550 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x57ae998ae5598f13 with lease ID 0x716ea05754cf13a4: Processing first storage report for DS-f57cbfb6-3bf7-4923-80fe-5d4ecae29e9a from datanode DatanodeRegistration(127.0.0.1:45613, datanodeUuid=9a84d457-85df-4ef8-9704-f07ec07f9ac5, infoPort=34135, infoSecurePort=0, ipcPort=34237, storageInfo=lv=-57;cid=testClusterID;nsid=1506381876;c=1733934366087)
2024-12-11T16:26:07,550 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x57ae998ae5598f13 with lease ID 0x716ea05754cf13a4: from storage DS-f57cbfb6-3bf7-4923-80fe-5d4ecae29e9a node DatanodeRegistration(127.0.0.1:45613, datanodeUuid=9a84d457-85df-4ef8-9704-f07ec07f9ac5, infoPort=34135, infoSecurePort=0, ipcPort=34237, storageInfo=lv=-57;cid=testClusterID;nsid=1506381876;c=1733934366087), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-11T16:26:07,681 WARN  [Thread-1983 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91bb4f8c-44ab-4329-ecbc-8f1e37d48355/cluster_8a707b7d-e55b-61a4-30e3-3320dbb3250b/data/data3/current/BP-470135049-172.17.0.2-1733934366087/current, will proceed with Du for space computation calculation, 
2024-12-11T16:26:07,681 WARN  [Thread-1984 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91bb4f8c-44ab-4329-ecbc-8f1e37d48355/cluster_8a707b7d-e55b-61a4-30e3-3320dbb3250b/data/data4/current/BP-470135049-172.17.0.2-1733934366087/current, will proceed with Du for space computation calculation, 
2024-12-11T16:26:07,698 WARN  [Thread-1959 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-11T16:26:07,700 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1b7421ea19359a01 with lease ID 0x716ea05754cf13a5: Processing first storage report for DS-0cd30412-ee07-4c3e-a0c1-33157088d4db from datanode DatanodeRegistration(127.0.0.1:42195, datanodeUuid=09486516-a0d5-489e-92e7-11e53adfb6cd, infoPort=35903, infoSecurePort=0, ipcPort=34987, storageInfo=lv=-57;cid=testClusterID;nsid=1506381876;c=1733934366087)
2024-12-11T16:26:07,700 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1b7421ea19359a01 with lease ID 0x716ea05754cf13a5: from storage DS-0cd30412-ee07-4c3e-a0c1-33157088d4db node DatanodeRegistration(127.0.0.1:42195, datanodeUuid=09486516-a0d5-489e-92e7-11e53adfb6cd, infoPort=35903, infoSecurePort=0, ipcPort=34987, storageInfo=lv=-57;cid=testClusterID;nsid=1506381876;c=1733934366087), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-11T16:26:07,700 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1b7421ea19359a01 with lease ID 0x716ea05754cf13a5: Processing first storage report for DS-6d22451b-353f-41a1-8c63-2d81c379ed69 from datanode DatanodeRegistration(127.0.0.1:42195, datanodeUuid=09486516-a0d5-489e-92e7-11e53adfb6cd, infoPort=35903, infoSecurePort=0, ipcPort=34987, storageInfo=lv=-57;cid=testClusterID;nsid=1506381876;c=1733934366087)
2024-12-11T16:26:07,700 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1b7421ea19359a01 with lease ID 0x716ea05754cf13a5: from storage DS-6d22451b-353f-41a1-8c63-2d81c379ed69 node DatanodeRegistration(127.0.0.1:42195, datanodeUuid=09486516-a0d5-489e-92e7-11e53adfb6cd, infoPort=35903, infoSecurePort=0, ipcPort=34987, storageInfo=lv=-57;cid=testClusterID;nsid=1506381876;c=1733934366087), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-11T16:26:07,805 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91bb4f8c-44ab-4329-ecbc-8f1e37d48355
2024-12-11T16:26:07,809 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91bb4f8c-44ab-4329-ecbc-8f1e37d48355/cluster_8a707b7d-e55b-61a4-30e3-3320dbb3250b/zookeeper_0, clientPort=51025, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91bb4f8c-44ab-4329-ecbc-8f1e37d48355/cluster_8a707b7d-e55b-61a4-30e3-3320dbb3250b/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91bb4f8c-44ab-4329-ecbc-8f1e37d48355/cluster_8a707b7d-e55b-61a4-30e3-3320dbb3250b/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-11T16:26:07,810 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=51025
2024-12-11T16:26:07,810 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T16:26:07,811 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T16:26:07,819 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42195 is added to blk_1073741825_1001 (size=7)
2024-12-11T16:26:07,820 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45613 is added to blk_1073741825_1001 (size=7)
2024-12-11T16:26:07,821 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f with version=8
2024-12-11T16:26:07,821 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/hbase-staging
2024-12-11T16:26:07,823 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/7446a65196a1:0 server-side Connection retries=45
2024-12-11T16:26:07,823 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-11T16:26:07,823 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-11T16:26:07,823 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-11T16:26:07,823 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-11T16:26:07,823 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-11T16:26:07,823 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-12-11T16:26:07,823 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-11T16:26:07,824 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39633
2024-12-11T16:26:07,825 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:39633 connecting to ZooKeeper ensemble=127.0.0.1:51025
2024-12-11T16:26:07,879 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:396330x0, quorum=127.0.0.1:51025, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-11T16:26:07,880 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:39633-0x10015c082dd0000 connected
2024-12-11T16:26:07,973 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T16:26:07,974 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T16:26:07,975 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39633-0x10015c082dd0000, quorum=127.0.0.1:51025, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-11T16:26:07,976 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f, hbase.cluster.distributed=false
2024-12-11T16:26:07,977 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39633-0x10015c082dd0000, quorum=127.0.0.1:51025, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-11T16:26:07,977 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39633
2024-12-11T16:26:07,977 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39633
2024-12-11T16:26:07,978 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39633
2024-12-11T16:26:07,978 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39633
2024-12-11T16:26:07,978 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39633
2024-12-11T16:26:07,993 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/7446a65196a1:0 server-side Connection retries=45
2024-12-11T16:26:07,993 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-11T16:26:07,993 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-11T16:26:07,993 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-11T16:26:07,993 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-11T16:26:07,993 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-11T16:26:07,993 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-11T16:26:07,993 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-11T16:26:07,994 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40919
2024-12-11T16:26:07,995 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40919 connecting to ZooKeeper ensemble=127.0.0.1:51025
2024-12-11T16:26:07,995 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T16:26:07,996 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T16:26:08,006 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:409190x0, quorum=127.0.0.1:51025, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-11T16:26:08,006 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:409190x0, quorum=127.0.0.1:51025, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-11T16:26:08,006 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40919-0x10015c082dd0001 connected
2024-12-11T16:26:08,006 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-11T16:26:08,007 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-11T16:26:08,008 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40919-0x10015c082dd0001, quorum=127.0.0.1:51025, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-11T16:26:08,008 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40919-0x10015c082dd0001, quorum=127.0.0.1:51025, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-11T16:26:08,009 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40919
2024-12-11T16:26:08,009 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40919
2024-12-11T16:26:08,009 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40919
2024-12-11T16:26:08,010 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40919
2024-12-11T16:26:08,010 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40919
2024-12-11T16:26:08,023 DEBUG [M:0;7446a65196a1:39633 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;7446a65196a1:39633
2024-12-11T16:26:08,023 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/7446a65196a1,39633,1733934367823
2024-12-11T16:26:08,031 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40919-0x10015c082dd0001, quorum=127.0.0.1:51025, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-11T16:26:08,031 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39633-0x10015c082dd0000, quorum=127.0.0.1:51025, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-11T16:26:08,031 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39633-0x10015c082dd0000, quorum=127.0.0.1:51025, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/7446a65196a1,39633,1733934367823
2024-12-11T16:26:08,039 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39633-0x10015c082dd0000, quorum=127.0.0.1:51025, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:26:08,039 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40919-0x10015c082dd0001, quorum=127.0.0.1:51025, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-11T16:26:08,039 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40919-0x10015c082dd0001, quorum=127.0.0.1:51025, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:26:08,040 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39633-0x10015c082dd0000, quorum=127.0.0.1:51025, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-11T16:26:08,040 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/7446a65196a1,39633,1733934367823 from backup master directory
2024-12-11T16:26:08,047 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39633-0x10015c082dd0000, quorum=127.0.0.1:51025, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/7446a65196a1,39633,1733934367823
2024-12-11T16:26:08,047 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40919-0x10015c082dd0001, quorum=127.0.0.1:51025, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-11T16:26:08,047 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39633-0x10015c082dd0000, quorum=127.0.0.1:51025, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-11T16:26:08,047 WARN  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-11T16:26:08,047 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=7446a65196a1,39633,1733934367823
2024-12-11T16:26:08,051 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/hbase.id] with ID: 0a362665-fbd1-43fe-b1c7-765c3cef1276
2024-12-11T16:26:08,051 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/.tmp/hbase.id
2024-12-11T16:26:08,057 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45613 is added to blk_1073741826_1002 (size=42)
2024-12-11T16:26:08,057 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42195 is added to blk_1073741826_1002 (size=42)
2024-12-11T16:26:08,058 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/.tmp/hbase.id]:[hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/hbase.id]
2024-12-11T16:26:08,068 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T16:26:08,068 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-12-11T16:26:08,069 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms.
2024-12-11T16:26:08,081 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40919-0x10015c082dd0001, quorum=127.0.0.1:51025, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:26:08,081 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39633-0x10015c082dd0000, quorum=127.0.0.1:51025, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:26:08,092 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42195 is added to blk_1073741827_1003 (size=196)
2024-12-11T16:26:08,092 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45613 is added to blk_1073741827_1003 (size=196)
2024-12-11T16:26:08,092 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-11T16:26:08,093 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-11T16:26:08,094 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-11T16:26:08,104 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45613 is added to blk_1073741828_1004 (size=1189)
2024-12-11T16:26:08,106 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/MasterData/data/master/store
2024-12-11T16:26:08,107 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42195 is added to blk_1073741828_1004 (size=1189)
2024-12-11T16:26:08,117 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42195 is added to blk_1073741829_1005 (size=34)
2024-12-11T16:26:08,117 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45613 is added to blk_1073741829_1005 (size=34)
2024-12-11T16:26:08,118 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T16:26:08,118 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-11T16:26:08,118 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T16:26:08,118 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T16:26:08,118 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-11T16:26:08,118 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T16:26:08,118 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T16:26:08,118 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733934368118Disabling compacts and flushes for region at 1733934368118Disabling writes for close at 1733934368118Writing region close event to WAL at 1733934368118Closed at 1733934368118
2024-12-11T16:26:08,119 WARN  [master/7446a65196a1:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/MasterData/data/master/store/.initializing
2024-12-11T16:26:08,119 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/MasterData/WALs/7446a65196a1,39633,1733934367823
2024-12-11T16:26:08,123 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7446a65196a1%2C39633%2C1733934367823, suffix=, logDir=hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/MasterData/WALs/7446a65196a1,39633,1733934367823, archiveDir=hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/MasterData/oldWALs, maxLogs=10
2024-12-11T16:26:08,123 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7446a65196a1%2C39633%2C1733934367823.1733934368123
2024-12-11T16:26:08,131 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/MasterData/WALs/7446a65196a1,39633,1733934367823/7446a65196a1%2C39633%2C1733934367823.1733934368123
2024-12-11T16:26:08,138 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35903:35903),(127.0.0.1/127.0.0.1:34135:34135)]
2024-12-11T16:26:08,140 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-11T16:26:08,140 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T16:26:08,140 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T16:26:08,140 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T16:26:08,142 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-11T16:26:08,144 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-11T16:26:08,144 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:26:08,144 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T16:26:08,144 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-11T16:26:08,146 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-11T16:26:08,146 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:26:08,146 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-11T16:26:08,146 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-11T16:26:08,148 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-11T16:26:08,148 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:26:08,148 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-11T16:26:08,148 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-11T16:26:08,149 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-11T16:26:08,149 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:26:08,150 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-11T16:26:08,150 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T16:26:08,151 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-11T16:26:08,151 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-11T16:26:08,152 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T16:26:08,152 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T16:26:08,153 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-11T16:26:08,154 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T16:26:08,159 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-11T16:26:08,160 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=738301, jitterRate=-0.06120258569717407}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-11T16:26:08,160 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1733934368140Initializing all the Stores at 1733934368141 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733934368141Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733934368142 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733934368142Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733934368142Cleaning up temporary data from old regions at 1733934368152 (+10 ms)Region opened successfully at 1733934368160 (+8 ms)
2024-12-11T16:26:08,161 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-11T16:26:08,163 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f0dc825, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7446a65196a1/172.17.0.2:0
2024-12-11T16:26:08,164 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-12-11T16:26:08,164 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-11T16:26:08,164 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-11T16:26:08,164 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-11T16:26:08,165 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-12-11T16:26:08,165 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-12-11T16:26:08,165 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-11T16:26:08,167 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-11T16:26:08,168 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39633-0x10015c082dd0000, quorum=127.0.0.1:51025, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-11T16:26:08,189 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-12-11T16:26:08,189 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-11T16:26:08,190 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39633-0x10015c082dd0000, quorum=127.0.0.1:51025, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-11T16:26:08,197 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-12-11T16:26:08,198 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-11T16:26:08,198 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39633-0x10015c082dd0000, quorum=127.0.0.1:51025, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-11T16:26:08,206 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-12-11T16:26:08,207 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39633-0x10015c082dd0000, quorum=127.0.0.1:51025, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-11T16:26:08,214 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-12-11T16:26:08,216 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39633-0x10015c082dd0000, quorum=127.0.0.1:51025, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-11T16:26:08,222 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-11T16:26:08,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40919-0x10015c082dd0001, quorum=127.0.0.1:51025, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-11T16:26:08,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39633-0x10015c082dd0000, quorum=127.0.0.1:51025, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-11T16:26:08,231 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40919-0x10015c082dd0001, quorum=127.0.0.1:51025, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:26:08,231 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39633-0x10015c082dd0000, quorum=127.0.0.1:51025, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:26:08,231 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=7446a65196a1,39633,1733934367823, sessionid=0x10015c082dd0000, setting cluster-up flag (Was=false)
2024-12-11T16:26:08,247 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40919-0x10015c082dd0001, quorum=127.0.0.1:51025, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:26:08,247 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39633-0x10015c082dd0000, quorum=127.0.0.1:51025, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:26:08,273 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-11T16:26:08,276 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=7446a65196a1,39633,1733934367823
2024-12-11T16:26:08,297 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39633-0x10015c082dd0000, quorum=127.0.0.1:51025, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:26:08,298 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40919-0x10015c082dd0001, quorum=127.0.0.1:51025, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:26:08,322 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-11T16:26:08,323 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=7446a65196a1,39633,1733934367823
2024-12-11T16:26:08,324 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-12-11T16:26:08,326 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-12-11T16:26:08,326 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-12-11T16:26:08,326 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-11T16:26:08,327 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 7446a65196a1,39633,1733934367823
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-11T16:26:08,328 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/7446a65196a1:0, corePoolSize=5, maxPoolSize=5
2024-12-11T16:26:08,328 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/7446a65196a1:0, corePoolSize=5, maxPoolSize=5
2024-12-11T16:26:08,328 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/7446a65196a1:0, corePoolSize=5, maxPoolSize=5
2024-12-11T16:26:08,328 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/7446a65196a1:0, corePoolSize=5, maxPoolSize=5
2024-12-11T16:26:08,328 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/7446a65196a1:0, corePoolSize=10, maxPoolSize=10
2024-12-11T16:26:08,328 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:26:08,328 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/7446a65196a1:0, corePoolSize=2, maxPoolSize=2
2024-12-11T16:26:08,328 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:26:08,331 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733934398331
2024-12-11T16:26:08,331 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-11T16:26:08,331 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-11T16:26:08,331 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-11T16:26:08,331 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-11T16:26:08,331 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-11T16:26:08,331 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-11T16:26:08,331 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-11T16:26:08,331 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-12-11T16:26:08,331 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-11T16:26:08,332 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-11T16:26:08,332 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-11T16:26:08,332 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-11T16:26:08,332 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-11T16:26:08,332 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-11T16:26:08,332 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/7446a65196a1:0:becomeActiveMaster-HFileCleaner.large.0-1733934368332,5,FailOnTimeoutGroup]
2024-12-11T16:26:08,332 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:26:08,332 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/7446a65196a1:0:becomeActiveMaster-HFileCleaner.small.0-1733934368332,5,FailOnTimeoutGroup]
2024-12-11T16:26:08,332 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-11T16:26:08,332 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-11T16:26:08,332 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-11T16:26:08,332 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-11T16:26:08,332 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-11T16:26:08,337 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45613 is added to blk_1073741831_1007 (size=1321)
2024-12-11T16:26:08,338 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42195 is added to blk_1073741831_1007 (size=1321)
2024-12-11T16:26:08,339 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-12-11T16:26:08,339 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f
2024-12-11T16:26:08,345 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42195 is added to blk_1073741832_1008 (size=32)
2024-12-11T16:26:08,345 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45613 is added to blk_1073741832_1008 (size=32)
2024-12-11T16:26:08,346 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T16:26:08,347 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-11T16:26:08,349 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-11T16:26:08,349 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:26:08,349 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T16:26:08,349 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-11T16:26:08,351 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-11T16:26:08,351 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:26:08,352 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T16:26:08,352 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-11T16:26:08,353 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-11T16:26:08,353 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:26:08,354 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T16:26:08,354 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-11T16:26:08,355 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-11T16:26:08,355 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:26:08,355 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T16:26:08,355 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-11T16:26:08,356 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/hbase/meta/1588230740
2024-12-11T16:26:08,356 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/hbase/meta/1588230740
2024-12-11T16:26:08,358 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-11T16:26:08,358 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-11T16:26:08,358 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-11T16:26:08,359 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-11T16:26:08,361 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-11T16:26:08,362 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=855967, jitterRate=0.08841900527477264}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-11T16:26:08,362 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1733934368346Initializing all the Stores at 1733934368347 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733934368347Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733934368347Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733934368347Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733934368347Cleaning up temporary data from old regions at 1733934368358 (+11 ms)Region opened successfully at 1733934368362 (+4 ms)
2024-12-11T16:26:08,362 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-11T16:26:08,362 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-11T16:26:08,362 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-11T16:26:08,362 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-11T16:26:08,362 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-11T16:26:08,363 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-11T16:26:08,363 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733934368362Disabling compacts and flushes for region at 1733934368362Disabling writes for close at 1733934368362Writing region close event to WAL at 1733934368363 (+1 ms)Closed at 1733934368363
2024-12-11T16:26:08,364 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-11T16:26:08,364 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-12-11T16:26:08,364 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-11T16:26:08,365 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-11T16:26:08,367 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-11T16:26:08,377 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:08,384 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:08,412 INFO  [RS:0;7446a65196a1:40919 {}] regionserver.HRegionServer(746): ClusterId : 0a362665-fbd1-43fe-b1c7-765c3cef1276
2024-12-11T16:26:08,412 DEBUG [RS:0;7446a65196a1:40919 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-11T16:26:08,429 DEBUG [RS:0;7446a65196a1:40919 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-11T16:26:08,429 DEBUG [RS:0;7446a65196a1:40919 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-11T16:26:08,440 DEBUG [RS:0;7446a65196a1:40919 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-11T16:26:08,440 DEBUG [RS:0;7446a65196a1:40919 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@742f4586, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7446a65196a1/172.17.0.2:0
2024-12-11T16:26:08,453 DEBUG [RS:0;7446a65196a1:40919 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;7446a65196a1:40919
2024-12-11T16:26:08,453 INFO  [RS:0;7446a65196a1:40919 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-12-11T16:26:08,453 INFO  [RS:0;7446a65196a1:40919 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-12-11T16:26:08,453 DEBUG [RS:0;7446a65196a1:40919 {}] regionserver.HRegionServer(832): About to register with Master.
2024-12-11T16:26:08,454 INFO  [RS:0;7446a65196a1:40919 {}] regionserver.HRegionServer(2659): reportForDuty to master=7446a65196a1,39633,1733934367823 with port=40919, startcode=1733934367992
2024-12-11T16:26:08,454 DEBUG [RS:0;7446a65196a1:40919 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-11T16:26:08,455 INFO  [HMaster-EventLoopGroup-14-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33509, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.6 (auth:SIMPLE), service=RegionServerStatusService
2024-12-11T16:26:08,456 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39633 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 7446a65196a1,40919,1733934367992
2024-12-11T16:26:08,456 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39633 {}] master.ServerManager(517): Registering regionserver=7446a65196a1,40919,1733934367992
2024-12-11T16:26:08,457 DEBUG [RS:0;7446a65196a1:40919 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f
2024-12-11T16:26:08,457 DEBUG [RS:0;7446a65196a1:40919 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44955
2024-12-11T16:26:08,457 DEBUG [RS:0;7446a65196a1:40919 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-12-11T16:26:08,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39633-0x10015c082dd0000, quorum=127.0.0.1:51025, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-11T16:26:08,464 DEBUG [RS:0;7446a65196a1:40919 {}] zookeeper.ZKUtil(111): regionserver:40919-0x10015c082dd0001, quorum=127.0.0.1:51025, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/7446a65196a1,40919,1733934367992
2024-12-11T16:26:08,464 WARN  [RS:0;7446a65196a1:40919 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-11T16:26:08,464 INFO  [RS:0;7446a65196a1:40919 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-11T16:26:08,464 DEBUG [RS:0;7446a65196a1:40919 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/WALs/7446a65196a1,40919,1733934367992
2024-12-11T16:26:08,465 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [7446a65196a1,40919,1733934367992]
2024-12-11T16:26:08,467 INFO  [RS:0;7446a65196a1:40919 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-11T16:26:08,468 INFO  [RS:0;7446a65196a1:40919 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-11T16:26:08,469 INFO  [RS:0;7446a65196a1:40919 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-11T16:26:08,469 INFO  [RS:0;7446a65196a1:40919 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-11T16:26:08,469 INFO  [RS:0;7446a65196a1:40919 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-12-11T16:26:08,470 INFO  [RS:0;7446a65196a1:40919 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-12-11T16:26:08,470 INFO  [RS:0;7446a65196a1:40919 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-11T16:26:08,470 DEBUG [RS:0;7446a65196a1:40919 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:26:08,470 DEBUG [RS:0;7446a65196a1:40919 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:26:08,470 DEBUG [RS:0;7446a65196a1:40919 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:26:08,470 DEBUG [RS:0;7446a65196a1:40919 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:26:08,470 DEBUG [RS:0;7446a65196a1:40919 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:26:08,470 DEBUG [RS:0;7446a65196a1:40919 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/7446a65196a1:0, corePoolSize=2, maxPoolSize=2
2024-12-11T16:26:08,470 DEBUG [RS:0;7446a65196a1:40919 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:26:08,470 DEBUG [RS:0;7446a65196a1:40919 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:26:08,470 DEBUG [RS:0;7446a65196a1:40919 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:26:08,470 DEBUG [RS:0;7446a65196a1:40919 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:26:08,470 DEBUG [RS:0;7446a65196a1:40919 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:26:08,470 DEBUG [RS:0;7446a65196a1:40919 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:26:08,470 DEBUG [RS:0;7446a65196a1:40919 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/7446a65196a1:0, corePoolSize=3, maxPoolSize=3
2024-12-11T16:26:08,470 DEBUG [RS:0;7446a65196a1:40919 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/7446a65196a1:0, corePoolSize=3, maxPoolSize=3
2024-12-11T16:26:08,471 INFO  [RS:0;7446a65196a1:40919 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-11T16:26:08,471 INFO  [RS:0;7446a65196a1:40919 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-11T16:26:08,471 INFO  [RS:0;7446a65196a1:40919 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-11T16:26:08,471 INFO  [RS:0;7446a65196a1:40919 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-11T16:26:08,471 INFO  [RS:0;7446a65196a1:40919 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-11T16:26:08,471 INFO  [RS:0;7446a65196a1:40919 {}] hbase.ChoreService(168): Chore ScheduledChore name=7446a65196a1,40919,1733934367992-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-11T16:26:08,485 INFO  [RS:0;7446a65196a1:40919 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-11T16:26:08,485 INFO  [RS:0;7446a65196a1:40919 {}] hbase.ChoreService(168): Chore ScheduledChore name=7446a65196a1,40919,1733934367992-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-11T16:26:08,485 INFO  [RS:0;7446a65196a1:40919 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T16:26:08,486 INFO  [RS:0;7446a65196a1:40919 {}] regionserver.Replication(171): 7446a65196a1,40919,1733934367992 started
2024-12-11T16:26:08,496 INFO  [RS:0;7446a65196a1:40919 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T16:26:08,496 INFO  [RS:0;7446a65196a1:40919 {}] regionserver.HRegionServer(1482): Serving as 7446a65196a1,40919,1733934367992, RpcServer on 7446a65196a1/172.17.0.2:40919, sessionid=0x10015c082dd0001
2024-12-11T16:26:08,496 DEBUG [RS:0;7446a65196a1:40919 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-11T16:26:08,496 DEBUG [RS:0;7446a65196a1:40919 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 7446a65196a1,40919,1733934367992
2024-12-11T16:26:08,496 DEBUG [RS:0;7446a65196a1:40919 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7446a65196a1,40919,1733934367992'
2024-12-11T16:26:08,496 DEBUG [RS:0;7446a65196a1:40919 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-11T16:26:08,497 DEBUG [RS:0;7446a65196a1:40919 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-11T16:26:08,498 DEBUG [RS:0;7446a65196a1:40919 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-11T16:26:08,498 DEBUG [RS:0;7446a65196a1:40919 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-11T16:26:08,498 DEBUG [RS:0;7446a65196a1:40919 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 7446a65196a1,40919,1733934367992
2024-12-11T16:26:08,498 DEBUG [RS:0;7446a65196a1:40919 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7446a65196a1,40919,1733934367992'
2024-12-11T16:26:08,498 DEBUG [RS:0;7446a65196a1:40919 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-11T16:26:08,498 DEBUG [RS:0;7446a65196a1:40919 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-11T16:26:08,498 DEBUG [RS:0;7446a65196a1:40919 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-11T16:26:08,498 INFO  [RS:0;7446a65196a1:40919 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-11T16:26:08,498 INFO  [RS:0;7446a65196a1:40919 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-11T16:26:08,517 WARN  [7446a65196a1:39633 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-12-11T16:26:08,602 INFO  [RS:0;7446a65196a1:40919 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7446a65196a1%2C40919%2C1733934367992, suffix=, logDir=hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/WALs/7446a65196a1,40919,1733934367992, archiveDir=hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/oldWALs, maxLogs=32
2024-12-11T16:26:08,603 INFO  [RS:0;7446a65196a1:40919 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7446a65196a1%2C40919%2C1733934367992.1733934368603
2024-12-11T16:26:08,612 INFO  [RS:0;7446a65196a1:40919 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/WALs/7446a65196a1,40919,1733934367992/7446a65196a1%2C40919%2C1733934367992.1733934368603
2024-12-11T16:26:08,613 DEBUG [RS:0;7446a65196a1:40919 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35903:35903),(127.0.0.1/127.0.0.1:34135:34135)]
2024-12-11T16:26:08,767 DEBUG [7446a65196a1:39633 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-12-11T16:26:08,768 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=7446a65196a1,40919,1733934367992
2024-12-11T16:26:08,769 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7446a65196a1,40919,1733934367992, state=OPENING
2024-12-11T16:26:08,797 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-11T16:26:08,806 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40919-0x10015c082dd0001, quorum=127.0.0.1:51025, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:26:08,806 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39633-0x10015c082dd0000, quorum=127.0.0.1:51025, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:26:08,807 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-11T16:26:08,807 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-11T16:26:08,807 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-11T16:26:08,808 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=7446a65196a1,40919,1733934367992}]
2024-12-11T16:26:08,963 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-11T16:26:08,965 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43779, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-11T16:26:08,968 INFO  [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-12-11T16:26:08,968 INFO  [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-11T16:26:08,969 INFO  [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7446a65196a1%2C40919%2C1733934367992.meta, suffix=.meta, logDir=hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/WALs/7446a65196a1,40919,1733934367992, archiveDir=hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/oldWALs, maxLogs=32
2024-12-11T16:26:08,969 INFO  [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 7446a65196a1%2C40919%2C1733934367992.meta.1733934368969.meta
2024-12-11T16:26:08,978 INFO  [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/WALs/7446a65196a1,40919,1733934367992/7446a65196a1%2C40919%2C1733934367992.meta.1733934368969.meta
2024-12-11T16:26:08,979 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35903:35903),(127.0.0.1/127.0.0.1:34135:34135)]
2024-12-11T16:26:08,980 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-11T16:26:08,980 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-11T16:26:08,980 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-11T16:26:08,980 INFO  [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-11T16:26:08,980 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-11T16:26:08,980 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T16:26:08,981 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-12-11T16:26:08,981 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-12-11T16:26:08,987 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-11T16:26:08,988 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-11T16:26:08,988 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:26:08,988 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T16:26:08,988 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-11T16:26:08,989 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-11T16:26:08,989 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:26:08,989 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T16:26:08,989 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-11T16:26:08,989 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-11T16:26:08,990 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:26:08,990 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T16:26:08,990 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-11T16:26:08,990 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-11T16:26:08,990 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:26:08,991 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T16:26:08,991 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-11T16:26:08,991 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/hbase/meta/1588230740
2024-12-11T16:26:08,992 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/hbase/meta/1588230740
2024-12-11T16:26:08,993 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-11T16:26:08,993 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-11T16:26:08,993 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-11T16:26:08,994 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-11T16:26:08,995 INFO  [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=778934, jitterRate=-0.009535074234008789}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-11T16:26:08,995 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-12-11T16:26:08,996 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1733934368981Writing region info on filesystem at 1733934368981Initializing all the Stores at 1733934368982 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733934368982Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733934368987 (+5 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733934368987Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733934368987Cleaning up temporary data from old regions at 1733934368993 (+6 ms)Running coprocessor post-open hooks at 1733934368995 (+2 ms)Region opened successfully at 1733934368995
2024-12-11T16:26:08,997 INFO  [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733934368963
2024-12-11T16:26:08,999 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-11T16:26:08,999 INFO  [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-12-11T16:26:08,999 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=7446a65196a1,40919,1733934367992
2024-12-11T16:26:09,000 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7446a65196a1,40919,1733934367992, state=OPEN
2024-12-11T16:26:09,056 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40919-0x10015c082dd0001, quorum=127.0.0.1:51025, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-11T16:26:09,056 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39633-0x10015c082dd0000, quorum=127.0.0.1:51025, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-11T16:26:09,056 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=7446a65196a1,40919,1733934367992
2024-12-11T16:26:09,056 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-11T16:26:09,056 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-11T16:26:09,061 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-12-11T16:26:09,061 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7446a65196a1,40919,1733934367992 in 249 msec
2024-12-11T16:26:09,065 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-12-11T16:26:09,065 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 697 msec
2024-12-11T16:26:09,066 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-11T16:26:09,066 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-12-11T16:26:09,067 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-11T16:26:09,067 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7446a65196a1,40919,1733934367992, seqNum=-1]
2024-12-11T16:26:09,068 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-11T16:26:09,069 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-15-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34695, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-11T16:26:09,075 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 748 msec
2024-12-11T16:26:09,075 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733934369075, completionTime=-1
2024-12-11T16:26:09,075 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-12-11T16:26:09,075 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-12-11T16:26:09,077 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-12-11T16:26:09,077 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733934429077
2024-12-11T16:26:09,077 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733934489077
2024-12-11T16:26:09,077 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 1 msec
2024-12-11T16:26:09,077 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7446a65196a1,39633,1733934367823-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-11T16:26:09,077 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7446a65196a1,39633,1733934367823-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T16:26:09,077 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7446a65196a1,39633,1733934367823-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T16:26:09,077 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-7446a65196a1:39633, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T16:26:09,077 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-11T16:26:09,077 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-12-11T16:26:09,079 DEBUG [master/7446a65196a1:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-12-11T16:26:09,081 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.034sec
2024-12-11T16:26:09,081 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-11T16:26:09,081 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-11T16:26:09,081 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-11T16:26:09,081 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-11T16:26:09,081 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-11T16:26:09,081 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7446a65196a1,39633,1733934367823-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-11T16:26:09,081 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7446a65196a1,39633,1733934367823-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-11T16:26:09,083 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-12-11T16:26:09,083 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-11T16:26:09,083 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7446a65196a1,39633,1733934367823-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T16:26:09,112 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@379c57e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-11T16:26:09,113 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7446a65196a1,39633,-1 for getting cluster id
2024-12-11T16:26:09,113 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-12-11T16:26:09,114 DEBUG [HMaster-EventLoopGroup-14-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a362665-fbd1-43fe-b1c7-765c3cef1276'
2024-12-11T16:26:09,115 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-12-11T16:26:09,115 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a362665-fbd1-43fe-b1c7-765c3cef1276"

2024-12-11T16:26:09,115 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5732c052, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-11T16:26:09,115 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7446a65196a1,39633,-1]
2024-12-11T16:26:09,116 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-12-11T16:26:09,116 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T16:26:09,117 INFO  [HMaster-EventLoopGroup-14-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34198, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-12-11T16:26:09,119 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3876b14b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-11T16:26:09,119 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-11T16:26:09,121 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7446a65196a1,40919,1733934367992, seqNum=-1]
2024-12-11T16:26:09,121 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-11T16:26:09,123 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35506, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-11T16:26:09,125 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=7446a65196a1,39633,1733934367823
2024-12-11T16:26:09,125 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T16:26:09,128 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-12-11T16:26:09,129 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-12-11T16:26:09,130 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 7446a65196a1,39633,1733934367823
2024-12-11T16:26:09,130 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2127f81c
2024-12-11T16:26:09,130 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-12-11T16:26:09,132 INFO  [HMaster-EventLoopGroup-14-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34210, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-12-11T16:26:09,132 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39633 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-12-11T16:26:09,132 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39633 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-12-11T16:26:09,133 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39633 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-11T16:26:09,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39633 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRolling
2024-12-11T16:26:09,135 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-12-11T16:26:09,135 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:26:09,135 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39633 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRolling"
 procId is: 4
2024-12-11T16:26:09,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-11T16:26:09,137 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-12-11T16:26:09,144 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42195 is added to blk_1073741835_1011 (size=381)
2024-12-11T16:26:09,144 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45613 is added to blk_1073741835_1011 (size=381)
2024-12-11T16:26:09,146 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c898cf8c864a19c76dae1a3f2f276629, NAME => 'TestLogRolling-testLogRolling,,1733934369132.c898cf8c864a19c76dae1a3f2f276629.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f
2024-12-11T16:26:09,151 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45613 is added to blk_1073741836_1012 (size=64)
2024-12-11T16:26:09,151 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42195 is added to blk_1073741836_1012 (size=64)
2024-12-11T16:26:09,152 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1733934369132.c898cf8c864a19c76dae1a3f2f276629.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T16:26:09,152 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1722): Closing c898cf8c864a19c76dae1a3f2f276629, disabling compactions & flushes
2024-12-11T16:26:09,152 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1733934369132.c898cf8c864a19c76dae1a3f2f276629.
2024-12-11T16:26:09,152 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1733934369132.c898cf8c864a19c76dae1a3f2f276629.
2024-12-11T16:26:09,152 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1733934369132.c898cf8c864a19c76dae1a3f2f276629. after waiting 0 ms
2024-12-11T16:26:09,152 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1733934369132.c898cf8c864a19c76dae1a3f2f276629.
2024-12-11T16:26:09,152 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1733934369132.c898cf8c864a19c76dae1a3f2f276629.
2024-12-11T16:26:09,152 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for c898cf8c864a19c76dae1a3f2f276629:
Waiting for close lock at 1733934369152Disabling compacts and flushes for region at 1733934369152Disabling writes for close at 1733934369152Writing region close event to WAL at 1733934369152Closed at 1733934369152
2024-12-11T16:26:09,154 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ADD_TO_META
2024-12-11T16:26:09,154 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRolling,,1733934369132.c898cf8c864a19c76dae1a3f2f276629.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1733934369154"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733934369154"}]},"ts":"1733934369154"}
2024-12-11T16:26:09,157 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-12-11T16:26:09,158 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-12-11T16:26:09,158 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733934369158"}]},"ts":"1733934369158"}
2024-12-11T16:26:09,160 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRolling, state=ENABLING in hbase:meta
2024-12-11T16:26:09,161 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=c898cf8c864a19c76dae1a3f2f276629, ASSIGN}]
2024-12-11T16:26:09,162 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=c898cf8c864a19c76dae1a3f2f276629, ASSIGN
2024-12-11T16:26:09,163 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=c898cf8c864a19c76dae1a3f2f276629, ASSIGN; state=OFFLINE, location=7446a65196a1,40919,1733934367992; forceNewPlan=false, retain=false
2024-12-11T16:26:09,314 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=c898cf8c864a19c76dae1a3f2f276629, regionState=OPENING, regionLocation=7446a65196a1,40919,1733934367992
2024-12-11T16:26:09,318 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=c898cf8c864a19c76dae1a3f2f276629, ASSIGN because future has completed
2024-12-11T16:26:09,319 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure c898cf8c864a19c76dae1a3f2f276629, server=7446a65196a1,40919,1733934367992}]
2024-12-11T16:26:09,378 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:09,384 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:09,483 INFO  [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,,1733934369132.c898cf8c864a19c76dae1a3f2f276629.
2024-12-11T16:26:09,483 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => c898cf8c864a19c76dae1a3f2f276629, NAME => 'TestLogRolling-testLogRolling,,1733934369132.c898cf8c864a19c76dae1a3f2f276629.', STARTKEY => '', ENDKEY => ''}
2024-12-11T16:26:09,484 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling c898cf8c864a19c76dae1a3f2f276629
2024-12-11T16:26:09,484 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1733934369132.c898cf8c864a19c76dae1a3f2f276629.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T16:26:09,484 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for c898cf8c864a19c76dae1a3f2f276629
2024-12-11T16:26:09,484 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for c898cf8c864a19c76dae1a3f2f276629
2024-12-11T16:26:09,486 INFO  [StoreOpener-c898cf8c864a19c76dae1a3f2f276629-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region c898cf8c864a19c76dae1a3f2f276629 
2024-12-11T16:26:09,487 INFO  [StoreOpener-c898cf8c864a19c76dae1a3f2f276629-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c898cf8c864a19c76dae1a3f2f276629 columnFamilyName info
2024-12-11T16:26:09,488 DEBUG [StoreOpener-c898cf8c864a19c76dae1a3f2f276629-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:26:09,488 INFO  [StoreOpener-c898cf8c864a19c76dae1a3f2f276629-1 {}] regionserver.HStore(327): Store=c898cf8c864a19c76dae1a3f2f276629/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-11T16:26:09,488 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for c898cf8c864a19c76dae1a3f2f276629
2024-12-11T16:26:09,489 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629
2024-12-11T16:26:09,489 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629
2024-12-11T16:26:09,489 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for c898cf8c864a19c76dae1a3f2f276629
2024-12-11T16:26:09,489 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for c898cf8c864a19c76dae1a3f2f276629
2024-12-11T16:26:09,491 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for c898cf8c864a19c76dae1a3f2f276629
2024-12-11T16:26:09,493 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-11T16:26:09,493 INFO  [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened c898cf8c864a19c76dae1a3f2f276629; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=703423, jitterRate=-0.10555163025856018}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-11T16:26:09,493 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c898cf8c864a19c76dae1a3f2f276629
2024-12-11T16:26:09,494 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for c898cf8c864a19c76dae1a3f2f276629:
Running coprocessor pre-open hook at 1733934369484Writing region info on filesystem at 1733934369484Initializing all the Stores at 1733934369486 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733934369486Cleaning up temporary data from old regions at 1733934369490 (+4 ms)Running coprocessor post-open hooks at 1733934369493 (+3 ms)Region opened successfully at 1733934369494 (+1 ms)
2024-12-11T16:26:09,494 INFO  [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,,1733934369132.c898cf8c864a19c76dae1a3f2f276629., pid=6, masterSystemTime=1733934369475
2024-12-11T16:26:09,497 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,,1733934369132.c898cf8c864a19c76dae1a3f2f276629.
2024-12-11T16:26:09,497 INFO  [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,,1733934369132.c898cf8c864a19c76dae1a3f2f276629.
2024-12-11T16:26:09,498 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=c898cf8c864a19c76dae1a3f2f276629, regionState=OPEN, openSeqNum=2, regionLocation=7446a65196a1,40919,1733934367992
2024-12-11T16:26:09,500 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure c898cf8c864a19c76dae1a3f2f276629, server=7446a65196a1,40919,1733934367992 because future has completed
2024-12-11T16:26:09,503 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-12-11T16:26:09,503 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure c898cf8c864a19c76dae1a3f2f276629, server=7446a65196a1,40919,1733934367992 in 182 msec
2024-12-11T16:26:09,505 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-12-11T16:26:09,505 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=c898cf8c864a19c76dae1a3f2f276629, ASSIGN in 342 msec
2024-12-11T16:26:09,506 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-12-11T16:26:09,506 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733934369506"}]},"ts":"1733934369506"}
2024-12-11T16:26:09,508 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRolling, state=ENABLED in hbase:meta
2024-12-11T16:26:09,509 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_POST_OPERATION
2024-12-11T16:26:09,511 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRolling in 376 msec
2024-12-11T16:26:10,107 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:26:10,107 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:26:10,108 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:26:10,108 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:26:10,109 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:26:10,109 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:26:10,111 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:26:10,111 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:26:10,127 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:26:10,127 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:26:10,127 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:26:10,128 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:26:10,128 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:26:10,128 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:26:10,130 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:26:10,130 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:26:10,130 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:26:10,132 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:26:10,378 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:10,385 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:10,636 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-11T16:26:10,638 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:26:10,639 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:26:10,639 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:26:10,640 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:26:10,640 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:26:10,641 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:26:10,642 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:26:10,642 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:26:10,660 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:26:10,660 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:26:10,660 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:26:10,661 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:26:10,661 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:26:10,661 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:26:10,663 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:26:10,664 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:26:10,664 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:26:10,666 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:26:11,379 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:11,386 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:12,381 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:12,387 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:13,381 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:13,388 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:14,382 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:14,388 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:14,467 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-12-11T16:26:14,468 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRolling'
2024-12-11T16:26:15,383 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:15,389 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:16,384 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:16,390 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:17,192 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-12-11T16:26:17,192 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-12-11T16:26:17,193 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-11T16:26:17,193 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-12-11T16:26:17,194 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-12-11T16:26:17,194 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer
2024-12-11T16:26:17,194 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling
2024-12-11T16:26:17,195 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling Metrics about Tables on a single HBase RegionServer
2024-12-11T16:26:17,385 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:17,391 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:18,386 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:18,392 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:19,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39633 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-11T16:26:19,211 INFO  [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRolling completed
2024-12-11T16:26:19,211 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRolling,, stopping at row=TestLogRolling-testLogRolling ,, for max=2147483647 with caching=100
2024-12-11T16:26:19,216 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRolling
2024-12-11T16:26:19,216 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRolling,,1733934369132.c898cf8c864a19c76dae1a3f2f276629.
2024-12-11T16:26:19,219 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testLogRolling,,1733934369132.c898cf8c864a19c76dae1a3f2f276629., hostname=7446a65196a1,40919,1733934367992, seqNum=2]
2024-12-11T16:26:19,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40919 {}] regionserver.HRegion(8855): Flush requested on c898cf8c864a19c76dae1a3f2f276629
2024-12-11T16:26:19,233 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c898cf8c864a19c76dae1a3f2f276629 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-11T16:26:19,250 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/.tmp/info/4c7fbf1eb14a44e8b43d4d1c526becde is 1080, key is row0001/info:/1733934379220/Put/seqid=0
2024-12-11T16:26:19,254 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42195 is added to blk_1073741837_1013 (size=12509)
2024-12-11T16:26:19,254 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45613 is added to blk_1073741837_1013 (size=12509)
2024-12-11T16:26:19,255 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/.tmp/info/4c7fbf1eb14a44e8b43d4d1c526becde
2024-12-11T16:26:19,261 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/.tmp/info/4c7fbf1eb14a44e8b43d4d1c526becde as hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/4c7fbf1eb14a44e8b43d4d1c526becde
2024-12-11T16:26:19,267 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/4c7fbf1eb14a44e8b43d4d1c526becde, entries=7, sequenceid=11, filesize=12.2 K
2024-12-11T16:26:19,268 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=18.91 KB/19368 for c898cf8c864a19c76dae1a3f2f276629 in 35ms, sequenceid=11, compaction requested=false
2024-12-11T16:26:19,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c898cf8c864a19c76dae1a3f2f276629:

2024-12-11T16:26:19,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40919 {}] regionserver.HRegion(8855): Flush requested on c898cf8c864a19c76dae1a3f2f276629
2024-12-11T16:26:19,268 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c898cf8c864a19c76dae1a3f2f276629 1/1 column families, dataSize=19.96 KB heapSize=21.63 KB
2024-12-11T16:26:19,272 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/.tmp/info/3216646b40814c2bac0cf0b797ff185a is 1080, key is row0008/info:/1733934379234/Put/seqid=0
2024-12-11T16:26:19,277 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45613 is added to blk_1073741838_1014 (size=25453)
2024-12-11T16:26:19,277 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42195 is added to blk_1073741838_1014 (size=25453)
2024-12-11T16:26:19,277 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=19.96 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/.tmp/info/3216646b40814c2bac0cf0b797ff185a
2024-12-11T16:26:19,284 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/.tmp/info/3216646b40814c2bac0cf0b797ff185a as hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/3216646b40814c2bac0cf0b797ff185a
2024-12-11T16:26:19,289 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/3216646b40814c2bac0cf0b797ff185a, entries=19, sequenceid=33, filesize=24.9 K
2024-12-11T16:26:19,290 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~19.96 KB/20444, heapSize ~21.61 KB/22128, currentSize=6.30 KB/6456 for c898cf8c864a19c76dae1a3f2f276629 in 22ms, sequenceid=33, compaction requested=false
2024-12-11T16:26:19,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c898cf8c864a19c76dae1a3f2f276629:

2024-12-11T16:26:19,290 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=37.1 K, sizeToCheck=16.0 K
2024-12-11T16:26:19,290 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-11T16:26:19,290 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/3216646b40814c2bac0cf0b797ff185a because midkey is the same as first or last row
2024-12-11T16:26:19,387 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:19,392 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:20,388 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:20,393 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:21,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40919 {}] regionserver.HRegion(8855): Flush requested on c898cf8c864a19c76dae1a3f2f276629
2024-12-11T16:26:21,285 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c898cf8c864a19c76dae1a3f2f276629 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-11T16:26:21,293 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/.tmp/info/407f97133a7943ad8d621dcb36a5e0ad is 1080, key is row0027/info:/1733934379269/Put/seqid=0
2024-12-11T16:26:21,297 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42195 is added to blk_1073741839_1015 (size=12509)
2024-12-11T16:26:21,297 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45613 is added to blk_1073741839_1015 (size=12509)
2024-12-11T16:26:21,298 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=43 (bloomFilter=true), to=hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/.tmp/info/407f97133a7943ad8d621dcb36a5e0ad
2024-12-11T16:26:21,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/.tmp/info/407f97133a7943ad8d621dcb36a5e0ad as hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/407f97133a7943ad8d621dcb36a5e0ad
2024-12-11T16:26:21,311 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/407f97133a7943ad8d621dcb36a5e0ad, entries=7, sequenceid=43, filesize=12.2 K
2024-12-11T16:26:21,312 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=11.56 KB/11836 for c898cf8c864a19c76dae1a3f2f276629 in 26ms, sequenceid=43, compaction requested=true
2024-12-11T16:26:21,312 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c898cf8c864a19c76dae1a3f2f276629:

2024-12-11T16:26:21,312 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=49.3 K, sizeToCheck=16.0 K
2024-12-11T16:26:21,312 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-11T16:26:21,312 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/3216646b40814c2bac0cf0b797ff185a because midkey is the same as first or last row
2024-12-11T16:26:21,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40919 {}] regionserver.HRegion(8855): Flush requested on c898cf8c864a19c76dae1a3f2f276629
2024-12-11T16:26:21,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c898cf8c864a19c76dae1a3f2f276629:info, priority=-2147483648, current under compaction store size is 1
2024-12-11T16:26:21,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-11T16:26:21,312 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-11T16:26:21,312 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c898cf8c864a19c76dae1a3f2f276629 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB
2024-12-11T16:26:21,314 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 50471 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-11T16:26:21,314 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.HStore(1541): c898cf8c864a19c76dae1a3f2f276629/info is initiating minor compaction (all files)
2024-12-11T16:26:21,314 INFO  [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c898cf8c864a19c76dae1a3f2f276629/info in TestLogRolling-testLogRolling,,1733934369132.c898cf8c864a19c76dae1a3f2f276629.
2024-12-11T16:26:21,314 INFO  [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/4c7fbf1eb14a44e8b43d4d1c526becde, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/3216646b40814c2bac0cf0b797ff185a, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/407f97133a7943ad8d621dcb36a5e0ad] into tmpdir=hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/.tmp, totalSize=49.3 K
2024-12-11T16:26:21,314 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c7fbf1eb14a44e8b43d4d1c526becde, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733934379220
2024-12-11T16:26:21,315 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3216646b40814c2bac0cf0b797ff185a, keycount=19, bloomtype=ROW, size=24.9 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1733934379234
2024-12-11T16:26:21,315 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 407f97133a7943ad8d621dcb36a5e0ad, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1733934379269
2024-12-11T16:26:21,317 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/.tmp/info/30a6db08014e46cea0fd85152bc115ed is 1080, key is row0034/info:/1733934381288/Put/seqid=0
2024-12-11T16:26:21,321 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42195 is added to blk_1073741840_1016 (size=17894)
2024-12-11T16:26:21,321 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45613 is added to blk_1073741840_1016 (size=17894)
2024-12-11T16:26:21,322 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/.tmp/info/30a6db08014e46cea0fd85152bc115ed
2024-12-11T16:26:21,328 INFO  [RS:0;7446a65196a1:40919-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c898cf8c864a19c76dae1a3f2f276629#info#compaction#59 average throughput is 16.93 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-11T16:26:21,329 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/.tmp/info/33b54346b60e4160ba6bd6651e9825d4 is 1080, key is row0001/info:/1733934379220/Put/seqid=0
2024-12-11T16:26:21,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/.tmp/info/30a6db08014e46cea0fd85152bc115ed as hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/30a6db08014e46cea0fd85152bc115ed
2024-12-11T16:26:21,336 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42195 is added to blk_1073741841_1017 (size=40670)
2024-12-11T16:26:21,337 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45613 is added to blk_1073741841_1017 (size=40670)
2024-12-11T16:26:21,337 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/30a6db08014e46cea0fd85152bc115ed, entries=12, sequenceid=58, filesize=17.5 K
2024-12-11T16:26:21,338 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=10.51 KB/10760 for c898cf8c864a19c76dae1a3f2f276629 in 25ms, sequenceid=58, compaction requested=false
2024-12-11T16:26:21,338 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c898cf8c864a19c76dae1a3f2f276629:

2024-12-11T16:26:21,338 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=66.8 K, sizeToCheck=16.0 K
2024-12-11T16:26:21,338 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-11T16:26:21,338 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/3216646b40814c2bac0cf0b797ff185a because midkey is the same as first or last row
2024-12-11T16:26:21,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40919 {}] regionserver.HRegion(8855): Flush requested on c898cf8c864a19c76dae1a3f2f276629
2024-12-11T16:26:21,340 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c898cf8c864a19c76dae1a3f2f276629 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB
2024-12-11T16:26:21,343 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/.tmp/info/33b54346b60e4160ba6bd6651e9825d4 as hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/33b54346b60e4160ba6bd6651e9825d4
2024-12-11T16:26:21,344 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/.tmp/info/1baf5eb7886242029a0ec11be3716803 is 1080, key is row0046/info:/1733934381313/Put/seqid=0
2024-12-11T16:26:21,349 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42195 is added to blk_1073741842_1018 (size=17894)
2024-12-11T16:26:21,349 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45613 is added to blk_1073741842_1018 (size=17894)
2024-12-11T16:26:21,350 INFO  [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c898cf8c864a19c76dae1a3f2f276629/info of c898cf8c864a19c76dae1a3f2f276629 into 33b54346b60e4160ba6bd6651e9825d4(size=39.7 K), total size for store is 57.2 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-11T16:26:21,350 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/.tmp/info/1baf5eb7886242029a0ec11be3716803
2024-12-11T16:26:21,350 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c898cf8c864a19c76dae1a3f2f276629:

2024-12-11T16:26:21,350 INFO  [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1733934369132.c898cf8c864a19c76dae1a3f2f276629., storeName=c898cf8c864a19c76dae1a3f2f276629/info, priority=13, startTime=1733934381312; duration=0sec
2024-12-11T16:26:21,350 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=57.2 K, sizeToCheck=16.0 K
2024-12-11T16:26:21,350 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-11T16:26:21,350 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/33b54346b60e4160ba6bd6651e9825d4 because midkey is the same as first or last row
2024-12-11T16:26:21,351 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=57.2 K, sizeToCheck=16.0 K
2024-12-11T16:26:21,351 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-11T16:26:21,351 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/33b54346b60e4160ba6bd6651e9825d4 because midkey is the same as first or last row
2024-12-11T16:26:21,351 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=57.2 K, sizeToCheck=16.0 K
2024-12-11T16:26:21,351 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-11T16:26:21,351 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/33b54346b60e4160ba6bd6651e9825d4 because midkey is the same as first or last row
2024-12-11T16:26:21,351 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-11T16:26:21,351 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c898cf8c864a19c76dae1a3f2f276629:info
2024-12-11T16:26:21,357 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/.tmp/info/1baf5eb7886242029a0ec11be3716803 as hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/1baf5eb7886242029a0ec11be3716803
2024-12-11T16:26:21,361 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/1baf5eb7886242029a0ec11be3716803, entries=12, sequenceid=73, filesize=17.5 K
2024-12-11T16:26:21,362 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=7.36 KB/7532 for c898cf8c864a19c76dae1a3f2f276629 in 22ms, sequenceid=73, compaction requested=true
2024-12-11T16:26:21,362 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c898cf8c864a19c76dae1a3f2f276629:

2024-12-11T16:26:21,362 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=74.7 K, sizeToCheck=16.0 K
2024-12-11T16:26:21,362 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-11T16:26:21,362 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/33b54346b60e4160ba6bd6651e9825d4 because midkey is the same as first or last row
2024-12-11T16:26:21,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c898cf8c864a19c76dae1a3f2f276629:info, priority=-2147483648, current under compaction store size is 1
2024-12-11T16:26:21,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-11T16:26:21,363 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-11T16:26:21,364 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 76458 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-11T16:26:21,364 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.HStore(1541): c898cf8c864a19c76dae1a3f2f276629/info is initiating minor compaction (all files)
2024-12-11T16:26:21,364 INFO  [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c898cf8c864a19c76dae1a3f2f276629/info in TestLogRolling-testLogRolling,,1733934369132.c898cf8c864a19c76dae1a3f2f276629.
2024-12-11T16:26:21,364 INFO  [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/33b54346b60e4160ba6bd6651e9825d4, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/30a6db08014e46cea0fd85152bc115ed, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/1baf5eb7886242029a0ec11be3716803] into tmpdir=hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/.tmp, totalSize=74.7 K
2024-12-11T16:26:21,364 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33b54346b60e4160ba6bd6651e9825d4, keycount=33, bloomtype=ROW, size=39.7 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1733934379220
2024-12-11T16:26:21,365 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30a6db08014e46cea0fd85152bc115ed, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733934381288
2024-12-11T16:26:21,365 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1baf5eb7886242029a0ec11be3716803, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1733934381313
2024-12-11T16:26:21,375 INFO  [RS:0;7446a65196a1:40919-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c898cf8c864a19c76dae1a3f2f276629#info#compaction#61 average throughput is 29.25 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-11T16:26:21,375 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/.tmp/info/80b00462f48e4f9da97a2b6b1f07596e is 1080, key is row0001/info:/1733934379220/Put/seqid=0
2024-12-11T16:26:21,379 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45613 is added to blk_1073741843_1019 (size=66689)
2024-12-11T16:26:21,380 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42195 is added to blk_1073741843_1019 (size=66689)
2024-12-11T16:26:21,385 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/.tmp/info/80b00462f48e4f9da97a2b6b1f07596e as hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/80b00462f48e4f9da97a2b6b1f07596e
2024-12-11T16:26:21,389 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:21,390 INFO  [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c898cf8c864a19c76dae1a3f2f276629/info of c898cf8c864a19c76dae1a3f2f276629 into 80b00462f48e4f9da97a2b6b1f07596e(size=65.1 K), total size for store is 65.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-11T16:26:21,390 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c898cf8c864a19c76dae1a3f2f276629:

2024-12-11T16:26:21,390 INFO  [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1733934369132.c898cf8c864a19c76dae1a3f2f276629., storeName=c898cf8c864a19c76dae1a3f2f276629/info, priority=13, startTime=1733934381362; duration=0sec
2024-12-11T16:26:21,390 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=65.1 K, sizeToCheck=16.0 K
2024-12-11T16:26:21,390 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-11T16:26:21,390 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/80b00462f48e4f9da97a2b6b1f07596e because midkey is the same as first or last row
2024-12-11T16:26:21,391 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=65.1 K, sizeToCheck=16.0 K
2024-12-11T16:26:21,391 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-11T16:26:21,391 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/80b00462f48e4f9da97a2b6b1f07596e because midkey is the same as first or last row
2024-12-11T16:26:21,391 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=65.1 K, sizeToCheck=16.0 K
2024-12-11T16:26:21,391 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-11T16:26:21,391 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/80b00462f48e4f9da97a2b6b1f07596e because midkey is the same as first or last row
2024-12-11T16:26:21,391 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-11T16:26:21,391 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c898cf8c864a19c76dae1a3f2f276629:info
2024-12-11T16:26:21,394 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:22,389 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:22,394 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:23,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40919 {}] regionserver.HRegion(8855): Flush requested on c898cf8c864a19c76dae1a3f2f276629
2024-12-11T16:26:23,360 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c898cf8c864a19c76dae1a3f2f276629 1/1 column families, dataSize=8.41 KB heapSize=9.25 KB
2024-12-11T16:26:23,367 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/.tmp/info/253c2fc97389463d9215d20a01cc69a5 is 1080, key is row0058/info:/1733934381341/Put/seqid=0
2024-12-11T16:26:23,374 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45613 is added to blk_1073741844_1020 (size=13586)
2024-12-11T16:26:23,375 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42195 is added to blk_1073741844_1020 (size=13586)
2024-12-11T16:26:23,375 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.41 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/.tmp/info/253c2fc97389463d9215d20a01cc69a5
2024-12-11T16:26:23,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/.tmp/info/253c2fc97389463d9215d20a01cc69a5 as hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/253c2fc97389463d9215d20a01cc69a5
2024-12-11T16:26:23,386 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/253c2fc97389463d9215d20a01cc69a5, entries=8, sequenceid=86, filesize=13.3 K
2024-12-11T16:26:23,387 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~8.41 KB/8608, heapSize ~9.23 KB/9456, currentSize=11.56 KB/11836 for c898cf8c864a19c76dae1a3f2f276629 in 28ms, sequenceid=86, compaction requested=false
2024-12-11T16:26:23,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c898cf8c864a19c76dae1a3f2f276629:

2024-12-11T16:26:23,387 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=78.4 K, sizeToCheck=16.0 K
2024-12-11T16:26:23,387 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-11T16:26:23,387 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/80b00462f48e4f9da97a2b6b1f07596e because midkey is the same as first or last row
2024-12-11T16:26:23,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40919 {}] regionserver.HRegion(8855): Flush requested on c898cf8c864a19c76dae1a3f2f276629
2024-12-11T16:26:23,389 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c898cf8c864a19c76dae1a3f2f276629 1/1 column families, dataSize=13.66 KB heapSize=14.88 KB
2024-12-11T16:26:23,390 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:23,393 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/.tmp/info/f6cf1657717142b0aa0164aac68f7c54 is 1080, key is row0066/info:/1733934383361/Put/seqid=0
2024-12-11T16:26:23,394 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:23,398 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45613 is added to blk_1073741845_1021 (size=18987)
2024-12-11T16:26:23,398 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42195 is added to blk_1073741845_1021 (size=18987)
2024-12-11T16:26:23,398 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.66 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/.tmp/info/f6cf1657717142b0aa0164aac68f7c54
2024-12-11T16:26:23,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/.tmp/info/f6cf1657717142b0aa0164aac68f7c54 as hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/f6cf1657717142b0aa0164aac68f7c54
2024-12-11T16:26:23,409 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/f6cf1657717142b0aa0164aac68f7c54, entries=13, sequenceid=102, filesize=18.5 K
2024-12-11T16:26:23,410 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~13.66 KB/13988, heapSize ~14.86 KB/15216, currentSize=9.46 KB/9684 for c898cf8c864a19c76dae1a3f2f276629 in 21ms, sequenceid=102, compaction requested=true
2024-12-11T16:26:23,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c898cf8c864a19c76dae1a3f2f276629:

2024-12-11T16:26:23,410 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=96.9 K, sizeToCheck=16.0 K
2024-12-11T16:26:23,410 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-11T16:26:23,410 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/80b00462f48e4f9da97a2b6b1f07596e because midkey is the same as first or last row
2024-12-11T16:26:23,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c898cf8c864a19c76dae1a3f2f276629:info, priority=-2147483648, current under compaction store size is 1
2024-12-11T16:26:23,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-11T16:26:23,410 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-11T16:26:23,411 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 99262 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-11T16:26:23,412 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.HStore(1541): c898cf8c864a19c76dae1a3f2f276629/info is initiating minor compaction (all files)
2024-12-11T16:26:23,412 INFO  [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c898cf8c864a19c76dae1a3f2f276629/info in TestLogRolling-testLogRolling,,1733934369132.c898cf8c864a19c76dae1a3f2f276629.
2024-12-11T16:26:23,412 INFO  [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/80b00462f48e4f9da97a2b6b1f07596e, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/253c2fc97389463d9215d20a01cc69a5, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/f6cf1657717142b0aa0164aac68f7c54] into tmpdir=hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/.tmp, totalSize=96.9 K
2024-12-11T16:26:23,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40919 {}] regionserver.HRegion(8855): Flush requested on c898cf8c864a19c76dae1a3f2f276629
2024-12-11T16:26:23,412 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c898cf8c864a19c76dae1a3f2f276629 1/1 column families, dataSize=10.51 KB heapSize=11.50 KB
2024-12-11T16:26:23,412 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80b00462f48e4f9da97a2b6b1f07596e, keycount=57, bloomtype=ROW, size=65.1 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1733934379220
2024-12-11T16:26:23,413 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 253c2fc97389463d9215d20a01cc69a5, keycount=8, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733934381341
2024-12-11T16:26:23,413 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6cf1657717142b0aa0164aac68f7c54, keycount=13, bloomtype=ROW, size=18.5 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1733934383361
2024-12-11T16:26:23,416 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/.tmp/info/6cb30f3d036a4b5a9bd44a22492e4318 is 1080, key is row0079/info:/1733934383390/Put/seqid=0
2024-12-11T16:26:23,421 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45613 is added to blk_1073741846_1022 (size=15740)
2024-12-11T16:26:23,421 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42195 is added to blk_1073741846_1022 (size=15740)
2024-12-11T16:26:23,421 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.51 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/.tmp/info/6cb30f3d036a4b5a9bd44a22492e4318
2024-12-11T16:26:23,426 INFO  [RS:0;7446a65196a1:40919-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c898cf8c864a19c76dae1a3f2f276629#info#compaction#65 average throughput is 26.68 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-11T16:26:23,427 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/.tmp/info/72b4c685a5da4939a2dce6f5710cfe3e is 1080, key is row0001/info:/1733934379220/Put/seqid=0
2024-12-11T16:26:23,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/.tmp/info/6cb30f3d036a4b5a9bd44a22492e4318 as hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/6cb30f3d036a4b5a9bd44a22492e4318
2024-12-11T16:26:23,430 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45613 is added to blk_1073741847_1023 (size=89485)
2024-12-11T16:26:23,430 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42195 is added to blk_1073741847_1023 (size=89485)
2024-12-11T16:26:23,434 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/6cb30f3d036a4b5a9bd44a22492e4318, entries=10, sequenceid=115, filesize=15.4 K
2024-12-11T16:26:23,435 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/.tmp/info/72b4c685a5da4939a2dce6f5710cfe3e as hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/72b4c685a5da4939a2dce6f5710cfe3e
2024-12-11T16:26:23,435 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~10.51 KB/10760, heapSize ~11.48 KB/11760, currentSize=8.41 KB/8608 for c898cf8c864a19c76dae1a3f2f276629 in 23ms, sequenceid=115, compaction requested=false
2024-12-11T16:26:23,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c898cf8c864a19c76dae1a3f2f276629:

2024-12-11T16:26:23,435 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=112.3 K, sizeToCheck=16.0 K
2024-12-11T16:26:23,435 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-11T16:26:23,435 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/80b00462f48e4f9da97a2b6b1f07596e because midkey is the same as first or last row
2024-12-11T16:26:23,441 INFO  [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c898cf8c864a19c76dae1a3f2f276629/info of c898cf8c864a19c76dae1a3f2f276629 into 72b4c685a5da4939a2dce6f5710cfe3e(size=87.4 K), total size for store is 102.8 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-11T16:26:23,441 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c898cf8c864a19c76dae1a3f2f276629:

2024-12-11T16:26:23,441 INFO  [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1733934369132.c898cf8c864a19c76dae1a3f2f276629., storeName=c898cf8c864a19c76dae1a3f2f276629/info, priority=13, startTime=1733934383410; duration=0sec
2024-12-11T16:26:23,441 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=102.8 K, sizeToCheck=16.0 K
2024-12-11T16:26:23,441 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-11T16:26:23,441 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=102.8 K, sizeToCheck=16.0 K
2024-12-11T16:26:23,441 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-11T16:26:23,441 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=102.8 K, sizeToCheck=16.0 K
2024-12-11T16:26:23,441 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-11T16:26:23,442 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.CompactSplit(239): Splitting TestLogRolling-testLogRolling,,1733934369132.c898cf8c864a19c76dae1a3f2f276629., compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-11T16:26:23,442 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-11T16:26:23,442 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c898cf8c864a19c76dae1a3f2f276629:info
2024-12-11T16:26:23,443 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39633 {}] assignment.AssignmentManager(1363): Split request from 7446a65196a1,40919,1733934367992, parent={ENCODED => c898cf8c864a19c76dae1a3f2f276629, NAME => 'TestLogRolling-testLogRolling,,1733934369132.c898cf8c864a19c76dae1a3f2f276629.', STARTKEY => '', ENDKEY => ''}, splitKey=row0062
2024-12-11T16:26:23,447 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39633 {}] assignment.SplitTableRegionProcedure(223): Splittable=true state=OPEN, location=7446a65196a1,40919,1733934367992
2024-12-11T16:26:23,450 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39633 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=c898cf8c864a19c76dae1a3f2f276629, daughterA=423277343d719e8aca173d7993f485d6, daughterB=f782a65af6e9ff7c34f274c17ab783a9
2024-12-11T16:26:23,451 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=c898cf8c864a19c76dae1a3f2f276629, daughterA=423277343d719e8aca173d7993f485d6, daughterB=f782a65af6e9ff7c34f274c17ab783a9
2024-12-11T16:26:23,451 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=c898cf8c864a19c76dae1a3f2f276629, daughterA=423277343d719e8aca173d7993f485d6, daughterB=f782a65af6e9ff7c34f274c17ab783a9
2024-12-11T16:26:23,451 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=c898cf8c864a19c76dae1a3f2f276629, daughterA=423277343d719e8aca173d7993f485d6, daughterB=f782a65af6e9ff7c34f274c17ab783a9
2024-12-11T16:26:23,492 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=c898cf8c864a19c76dae1a3f2f276629, UNASSIGN}]
2024-12-11T16:26:23,494 INFO  [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=c898cf8c864a19c76dae1a3f2f276629, UNASSIGN
2024-12-11T16:26:23,497 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=c898cf8c864a19c76dae1a3f2f276629, regionState=CLOSING, regionLocation=7446a65196a1,40919,1733934367992
2024-12-11T16:26:23,499 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=c898cf8c864a19c76dae1a3f2f276629, UNASSIGN because future has completed
2024-12-11T16:26:23,500 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: true: evictOnSplit: true: evictOnClose: false
2024-12-11T16:26:23,500 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; CloseRegionProcedure c898cf8c864a19c76dae1a3f2f276629, server=7446a65196a1,40919,1733934367992}]
2024-12-11T16:26:23,659 INFO  [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(122): Close c898cf8c864a19c76dae1a3f2f276629
2024-12-11T16:26:23,659 DEBUG [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(136): Unassign region: split region: true: evictCache: true
2024-12-11T16:26:23,660 DEBUG [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1722): Closing c898cf8c864a19c76dae1a3f2f276629, disabling compactions & flushes
2024-12-11T16:26:23,660 INFO  [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1733934369132.c898cf8c864a19c76dae1a3f2f276629.
2024-12-11T16:26:23,660 DEBUG [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1733934369132.c898cf8c864a19c76dae1a3f2f276629.
2024-12-11T16:26:23,660 DEBUG [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1733934369132.c898cf8c864a19c76dae1a3f2f276629. after waiting 0 ms
2024-12-11T16:26:23,660 DEBUG [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1733934369132.c898cf8c864a19c76dae1a3f2f276629.
2024-12-11T16:26:23,660 INFO  [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(2902): Flushing c898cf8c864a19c76dae1a3f2f276629 1/1 column families, dataSize=8.41 KB heapSize=9.25 KB
2024-12-11T16:26:23,667 DEBUG [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/.tmp/info/da090a8ba1e84fd3be7e4bfa6f99d5c8 is 1080, key is row0089/info:/1733934383413/Put/seqid=0
2024-12-11T16:26:23,673 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42195 is added to blk_1073741848_1024 (size=13586)
2024-12-11T16:26:23,674 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45613 is added to blk_1073741848_1024 (size=13586)
2024-12-11T16:26:23,674 INFO  [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.41 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/.tmp/info/da090a8ba1e84fd3be7e4bfa6f99d5c8
2024-12-11T16:26:23,680 DEBUG [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/.tmp/info/da090a8ba1e84fd3be7e4bfa6f99d5c8 as hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/da090a8ba1e84fd3be7e4bfa6f99d5c8
2024-12-11T16:26:23,686 INFO  [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/da090a8ba1e84fd3be7e4bfa6f99d5c8, entries=8, sequenceid=127, filesize=13.3 K
2024-12-11T16:26:23,687 INFO  [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(3140): Finished flush of dataSize ~8.41 KB/8608, heapSize ~9.23 KB/9456, currentSize=0 B/0 for c898cf8c864a19c76dae1a3f2f276629 in 26ms, sequenceid=127, compaction requested=true
2024-12-11T16:26:23,688 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733934369132.c898cf8c864a19c76dae1a3f2f276629.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/4c7fbf1eb14a44e8b43d4d1c526becde, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/3216646b40814c2bac0cf0b797ff185a, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/33b54346b60e4160ba6bd6651e9825d4, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/407f97133a7943ad8d621dcb36a5e0ad, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/30a6db08014e46cea0fd85152bc115ed, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/80b00462f48e4f9da97a2b6b1f07596e, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/1baf5eb7886242029a0ec11be3716803, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/253c2fc97389463d9215d20a01cc69a5, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/f6cf1657717142b0aa0164aac68f7c54] to archive
2024-12-11T16:26:23,689 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733934369132.c898cf8c864a19c76dae1a3f2f276629.-1 {}] backup.HFileArchiver(363): Archiving compacted files.
2024-12-11T16:26:23,691 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/3216646b40814c2bac0cf0b797ff185a to hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/archive/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/3216646b40814c2bac0cf0b797ff185a
2024-12-11T16:26:23,691 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/33b54346b60e4160ba6bd6651e9825d4 to hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/archive/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/33b54346b60e4160ba6bd6651e9825d4
2024-12-11T16:26:23,691 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/4c7fbf1eb14a44e8b43d4d1c526becde to hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/archive/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/4c7fbf1eb14a44e8b43d4d1c526becde
2024-12-11T16:26:23,692 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/407f97133a7943ad8d621dcb36a5e0ad to hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/archive/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/407f97133a7943ad8d621dcb36a5e0ad
2024-12-11T16:26:23,692 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/80b00462f48e4f9da97a2b6b1f07596e to hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/archive/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/80b00462f48e4f9da97a2b6b1f07596e
2024-12-11T16:26:23,692 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/253c2fc97389463d9215d20a01cc69a5 to hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/archive/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/253c2fc97389463d9215d20a01cc69a5
2024-12-11T16:26:23,692 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/30a6db08014e46cea0fd85152bc115ed to hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/archive/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/30a6db08014e46cea0fd85152bc115ed
2024-12-11T16:26:23,692 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/1baf5eb7886242029a0ec11be3716803 to hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/archive/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/1baf5eb7886242029a0ec11be3716803
2024-12-11T16:26:23,693 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/f6cf1657717142b0aa0164aac68f7c54 to hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/archive/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/f6cf1657717142b0aa0164aac68f7c54
2024-12-11T16:26:23,699 DEBUG [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/recovered.edits/130.seqid, newMaxSeqId=130, maxSeqId=1
2024-12-11T16:26:23,700 INFO  [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1733934369132.c898cf8c864a19c76dae1a3f2f276629.
2024-12-11T16:26:23,700 DEBUG [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1676): Region close journal for c898cf8c864a19c76dae1a3f2f276629:
Waiting for close lock at 1733934383660Running coprocessor pre-close hooks at 1733934383660Disabling compacts and flushes for region at 1733934383660Disabling writes for close at 1733934383660Obtaining lock to block concurrent updates at 1733934383660Preparing flush snapshotting stores in c898cf8c864a19c76dae1a3f2f276629 at 1733934383660Finished memstore snapshotting TestLogRolling-testLogRolling,,1733934369132.c898cf8c864a19c76dae1a3f2f276629., syncing WAL and waiting on mvcc, flushsize=dataSize=8608, getHeapSize=9456, getOffHeapSize=0, getCellsCount=8 at 1733934383661 (+1 ms)Flushing stores of TestLogRolling-testLogRolling,,1733934369132.c898cf8c864a19c76dae1a3f2f276629. at 1733934383662 (+1 ms)Flushing c898cf8c864a19c76dae1a3f2f276629/info: creating writer at 1733934383662Flushing c898cf8c864a19c76dae1a3f2f276629/info: appending metadata at 1733934383666 (+4 ms)Flushing c898cf8c864a19c76dae1a3f2f276629/info: closing flushed file at 1733934383666Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3e2a0bf2: reopening flushed file at 1733934383679 (+13 ms)Finished flush of dataSize ~8.41 KB/8608, heapSize ~9.23 KB/9456, currentSize=0 B/0 for c898cf8c864a19c76dae1a3f2f276629 in 26ms, sequenceid=127, compaction requested=true at 1733934383687 (+8 ms)Writing region close event to WAL at 1733934383695 (+8 ms)Running coprocessor post-close hooks at 1733934383699 (+4 ms)Closed at 1733934383700 (+1 ms)
2024-12-11T16:26:23,702 INFO  [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(157): Closed c898cf8c864a19c76dae1a3f2f276629
2024-12-11T16:26:23,702 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=c898cf8c864a19c76dae1a3f2f276629, regionState=CLOSED
2024-12-11T16:26:23,704 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; CloseRegionProcedure c898cf8c864a19c76dae1a3f2f276629, server=7446a65196a1,40919,1733934367992 because future has completed
2024-12-11T16:26:23,708 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8
2024-12-11T16:26:23,708 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; CloseRegionProcedure c898cf8c864a19c76dae1a3f2f276629, server=7446a65196a1,40919,1733934367992 in 205 msec
2024-12-11T16:26:23,710 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7
2024-12-11T16:26:23,710 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=c898cf8c864a19c76dae1a3f2f276629, UNASSIGN in 216 msec
2024-12-11T16:26:23,720 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:26:23,725 INFO  [PEWorker-4 {}] assignment.SplitTableRegionProcedure(728): pid=7 splitting 3 storefiles, region=c898cf8c864a19c76dae1a3f2f276629, threads=3
2024-12-11T16:26:23,728 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/6cb30f3d036a4b5a9bd44a22492e4318 for region: c898cf8c864a19c76dae1a3f2f276629
2024-12-11T16:26:23,728 DEBUG [StoreFileSplitter-pool-1 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/72b4c685a5da4939a2dce6f5710cfe3e for region: c898cf8c864a19c76dae1a3f2f276629
2024-12-11T16:26:23,728 DEBUG [StoreFileSplitter-pool-2 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/da090a8ba1e84fd3be7e4bfa6f99d5c8 for region: c898cf8c864a19c76dae1a3f2f276629
2024-12-11T16:26:23,737 DEBUG [StoreFileSplitter-pool-0 {}] regionserver.HRegionFileSystem(650): Will create HFileLink file for hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/6cb30f3d036a4b5a9bd44a22492e4318, top=true
2024-12-11T16:26:23,739 DEBUG [StoreFileSplitter-pool-2 {}] regionserver.HRegionFileSystem(650): Will create HFileLink file for hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/da090a8ba1e84fd3be7e4bfa6f99d5c8, top=true
2024-12-11T16:26:23,742 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45613 is added to blk_1073741849_1025 (size=27)
2024-12-11T16:26:23,742 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42195 is added to blk_1073741849_1025 (size=27)
2024-12-11T16:26:23,744 INFO  [StoreFileSplitter-pool-0 {}] regionserver.HRegionFileSystem(691): Created linkFile:hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/TestLogRolling-testLogRolling=c898cf8c864a19c76dae1a3f2f276629-6cb30f3d036a4b5a9bd44a22492e4318 for child: f782a65af6e9ff7c34f274c17ab783a9, parent: c898cf8c864a19c76dae1a3f2f276629
2024-12-11T16:26:23,744 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/6cb30f3d036a4b5a9bd44a22492e4318 for region: c898cf8c864a19c76dae1a3f2f276629
2024-12-11T16:26:23,745 INFO  [StoreFileSplitter-pool-2 {}] regionserver.HRegionFileSystem(691): Created linkFile:hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/TestLogRolling-testLogRolling=c898cf8c864a19c76dae1a3f2f276629-da090a8ba1e84fd3be7e4bfa6f99d5c8 for child: f782a65af6e9ff7c34f274c17ab783a9, parent: c898cf8c864a19c76dae1a3f2f276629
2024-12-11T16:26:23,745 DEBUG [StoreFileSplitter-pool-2 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/da090a8ba1e84fd3be7e4bfa6f99d5c8 for region: c898cf8c864a19c76dae1a3f2f276629
2024-12-11T16:26:23,751 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42195 is added to blk_1073741850_1026 (size=27)
2024-12-11T16:26:23,751 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45613 is added to blk_1073741850_1026 (size=27)
2024-12-11T16:26:23,751 DEBUG [StoreFileSplitter-pool-1 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/72b4c685a5da4939a2dce6f5710cfe3e for region: c898cf8c864a19c76dae1a3f2f276629
2024-12-11T16:26:23,753 DEBUG [PEWorker-4 {}] assignment.SplitTableRegionProcedure(802): pid=7 split storefiles for region c898cf8c864a19c76dae1a3f2f276629 Daughter A: [hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/423277343d719e8aca173d7993f485d6/info/72b4c685a5da4939a2dce6f5710cfe3e.c898cf8c864a19c76dae1a3f2f276629] storefiles, Daughter B: [hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/TestLogRolling-testLogRolling=c898cf8c864a19c76dae1a3f2f276629-6cb30f3d036a4b5a9bd44a22492e4318, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/72b4c685a5da4939a2dce6f5710cfe3e.c898cf8c864a19c76dae1a3f2f276629, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/TestLogRolling-testLogRolling=c898cf8c864a19c76dae1a3f2f276629-da090a8ba1e84fd3be7e4bfa6f99d5c8] storefiles.
2024-12-11T16:26:23,759 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42195 is added to blk_1073741851_1027 (size=71)
2024-12-11T16:26:23,759 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45613 is added to blk_1073741851_1027 (size=71)
2024-12-11T16:26:23,761 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:26:23,771 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45613 is added to blk_1073741852_1028 (size=71)
2024-12-11T16:26:23,771 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42195 is added to blk_1073741852_1028 (size=71)
2024-12-11T16:26:23,772 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:26:23,781 DEBUG [PEWorker-4 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/423277343d719e8aca173d7993f485d6/recovered.edits/130.seqid, newMaxSeqId=130, maxSeqId=-1
2024-12-11T16:26:23,782 DEBUG [PEWorker-4 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/recovered.edits/130.seqid, newMaxSeqId=130, maxSeqId=-1
2024-12-11T16:26:23,785 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1733934369132.c898cf8c864a19c76dae1a3f2f276629.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1733934383784"},{"qualifier":"splitA","vlen":70,"tag":[],"timestamp":"1733934383784"},{"qualifier":"splitB","vlen":70,"tag":[],"timestamp":"1733934383784"}]},"ts":"1733934383784"}
2024-12-11T16:26:23,785 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1733934383447.423277343d719e8aca173d7993f485d6.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1733934383784"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733934383784"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1733934383784"}]},"ts":"1733934383784"}
2024-12-11T16:26:23,785 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,row0062,1733934383447.f782a65af6e9ff7c34f274c17ab783a9.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1733934383784"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733934383784"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1733934383784"}]},"ts":"1733934383784"}
2024-12-11T16:26:23,800 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=423277343d719e8aca173d7993f485d6, ASSIGN}, {pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=f782a65af6e9ff7c34f274c17ab783a9, ASSIGN}]
2024-12-11T16:26:23,801 INFO  [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=f782a65af6e9ff7c34f274c17ab783a9, ASSIGN
2024-12-11T16:26:23,801 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=423277343d719e8aca173d7993f485d6, ASSIGN
2024-12-11T16:26:23,802 INFO  [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=f782a65af6e9ff7c34f274c17ab783a9, ASSIGN; state=SPLITTING_NEW, location=7446a65196a1,40919,1733934367992; forceNewPlan=false, retain=false
2024-12-11T16:26:23,802 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=423277343d719e8aca173d7993f485d6, ASSIGN; state=SPLITTING_NEW, location=7446a65196a1,40919,1733934367992; forceNewPlan=false, retain=false
2024-12-11T16:26:23,954 INFO  [PEWorker-2 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=f782a65af6e9ff7c34f274c17ab783a9, regionState=OPENING, regionLocation=7446a65196a1,40919,1733934367992
2024-12-11T16:26:23,954 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=423277343d719e8aca173d7993f485d6, regionState=OPENING, regionLocation=7446a65196a1,40919,1733934367992
2024-12-11T16:26:23,959 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=f782a65af6e9ff7c34f274c17ab783a9, ASSIGN because future has completed
2024-12-11T16:26:23,960 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure f782a65af6e9ff7c34f274c17ab783a9, server=7446a65196a1,40919,1733934367992}]
2024-12-11T16:26:23,961 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=423277343d719e8aca173d7993f485d6, ASSIGN because future has completed
2024-12-11T16:26:23,962 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=13, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 423277343d719e8aca173d7993f485d6, server=7446a65196a1,40919,1733934367992}]
2024-12-11T16:26:24,121 INFO  [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,row0062,1733934383447.f782a65af6e9ff7c34f274c17ab783a9.
2024-12-11T16:26:24,121 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => f782a65af6e9ff7c34f274c17ab783a9, NAME => 'TestLogRolling-testLogRolling,row0062,1733934383447.f782a65af6e9ff7c34f274c17ab783a9.', STARTKEY => 'row0062', ENDKEY => ''}
2024-12-11T16:26:24,121 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling f782a65af6e9ff7c34f274c17ab783a9
2024-12-11T16:26:24,122 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,row0062,1733934383447.f782a65af6e9ff7c34f274c17ab783a9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T16:26:24,122 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for f782a65af6e9ff7c34f274c17ab783a9
2024-12-11T16:26:24,122 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for f782a65af6e9ff7c34f274c17ab783a9
2024-12-11T16:26:24,124 INFO  [StoreOpener-f782a65af6e9ff7c34f274c17ab783a9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region f782a65af6e9ff7c34f274c17ab783a9 
2024-12-11T16:26:24,125 INFO  [StoreOpener-f782a65af6e9ff7c34f274c17ab783a9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f782a65af6e9ff7c34f274c17ab783a9 columnFamilyName info
2024-12-11T16:26:24,125 DEBUG [StoreOpener-f782a65af6e9ff7c34f274c17ab783a9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:26:24,140 DEBUG [StoreOpener-f782a65af6e9ff7c34f274c17ab783a9-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/72b4c685a5da4939a2dce6f5710cfe3e.c898cf8c864a19c76dae1a3f2f276629->hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/72b4c685a5da4939a2dce6f5710cfe3e-top
2024-12-11T16:26:24,145 DEBUG [StoreOpener-f782a65af6e9ff7c34f274c17ab783a9-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/TestLogRolling-testLogRolling=c898cf8c864a19c76dae1a3f2f276629-6cb30f3d036a4b5a9bd44a22492e4318
2024-12-11T16:26:24,150 DEBUG [StoreOpener-f782a65af6e9ff7c34f274c17ab783a9-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/TestLogRolling-testLogRolling=c898cf8c864a19c76dae1a3f2f276629-da090a8ba1e84fd3be7e4bfa6f99d5c8
2024-12-11T16:26:24,150 INFO  [StoreOpener-f782a65af6e9ff7c34f274c17ab783a9-1 {}] regionserver.HStore(327): Store=f782a65af6e9ff7c34f274c17ab783a9/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-11T16:26:24,150 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for f782a65af6e9ff7c34f274c17ab783a9
2024-12-11T16:26:24,151 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9
2024-12-11T16:26:24,152 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9
2024-12-11T16:26:24,153 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for f782a65af6e9ff7c34f274c17ab783a9
2024-12-11T16:26:24,153 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for f782a65af6e9ff7c34f274c17ab783a9
2024-12-11T16:26:24,155 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for f782a65af6e9ff7c34f274c17ab783a9
2024-12-11T16:26:24,156 INFO  [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened f782a65af6e9ff7c34f274c17ab783a9; next sequenceid=131; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=703443, jitterRate=-0.10552661120891571}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-11T16:26:24,156 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f782a65af6e9ff7c34f274c17ab783a9
2024-12-11T16:26:24,157 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for f782a65af6e9ff7c34f274c17ab783a9:
Running coprocessor pre-open hook at 1733934384122Writing region info on filesystem at 1733934384122Initializing all the Stores at 1733934384123 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733934384123Cleaning up temporary data from old regions at 1733934384153 (+30 ms)Running coprocessor post-open hooks at 1733934384156 (+3 ms)Region opened successfully at 1733934384157 (+1 ms)
2024-12-11T16:26:24,158 INFO  [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,row0062,1733934383447.f782a65af6e9ff7c34f274c17ab783a9., pid=12, masterSystemTime=1733934384113
2024-12-11T16:26:24,158 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.CompactSplit(403): Add compact mark for store f782a65af6e9ff7c34f274c17ab783a9:info, priority=-2147483648, current under compaction store size is 1
2024-12-11T16:26:24,158 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-11T16:26:24,158 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-11T16:26:24,159 INFO  [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.HStore(1527): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,row0062,1733934383447.f782a65af6e9ff7c34f274c17ab783a9.
2024-12-11T16:26:24,159 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.HStore(1541): f782a65af6e9ff7c34f274c17ab783a9/info is initiating minor compaction (all files)
2024-12-11T16:26:24,160 INFO  [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f782a65af6e9ff7c34f274c17ab783a9/info in TestLogRolling-testLogRolling,row0062,1733934383447.f782a65af6e9ff7c34f274c17ab783a9.
2024-12-11T16:26:24,160 INFO  [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/72b4c685a5da4939a2dce6f5710cfe3e.c898cf8c864a19c76dae1a3f2f276629->hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/72b4c685a5da4939a2dce6f5710cfe3e-top, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/TestLogRolling-testLogRolling=c898cf8c864a19c76dae1a3f2f276629-6cb30f3d036a4b5a9bd44a22492e4318, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/TestLogRolling-testLogRolling=c898cf8c864a19c76dae1a3f2f276629-da090a8ba1e84fd3be7e4bfa6f99d5c8] into tmpdir=hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/.tmp, totalSize=116.0 K
2024-12-11T16:26:24,160 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,row0062,1733934383447.f782a65af6e9ff7c34f274c17ab783a9.
2024-12-11T16:26:24,160 INFO  [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,row0062,1733934383447.f782a65af6e9ff7c34f274c17ab783a9.
2024-12-11T16:26:24,161 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72b4c685a5da4939a2dce6f5710cfe3e.c898cf8c864a19c76dae1a3f2f276629, keycount=39, bloomtype=ROW, size=87.4 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1733934379220
2024-12-11T16:26:24,161 INFO  [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,,1733934383447.423277343d719e8aca173d7993f485d6.
2024-12-11T16:26:24,161 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7752): Opening region: {ENCODED => 423277343d719e8aca173d7993f485d6, NAME => 'TestLogRolling-testLogRolling,,1733934383447.423277343d719e8aca173d7993f485d6.', STARTKEY => '', ENDKEY => 'row0062'}
2024-12-11T16:26:24,161 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] compactions.Compactor(225): Compacting TestLogRolling-testLogRolling=c898cf8c864a19c76dae1a3f2f276629-6cb30f3d036a4b5a9bd44a22492e4318, keycount=10, bloomtype=ROW, size=15.4 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1733934383390
2024-12-11T16:26:24,161 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling 423277343d719e8aca173d7993f485d6
2024-12-11T16:26:24,161 INFO  [PEWorker-2 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=f782a65af6e9ff7c34f274c17ab783a9, regionState=OPEN, openSeqNum=131, regionLocation=7446a65196a1,40919,1733934367992
2024-12-11T16:26:24,161 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1733934383447.423277343d719e8aca173d7993f485d6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T16:26:24,161 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] compactions.Compactor(225): Compacting TestLogRolling-testLogRolling=c898cf8c864a19c76dae1a3f2f276629-da090a8ba1e84fd3be7e4bfa6f99d5c8, keycount=8, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1733934383413
2024-12-11T16:26:24,161 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7794): checking encryption for 423277343d719e8aca173d7993f485d6
2024-12-11T16:26:24,161 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7797): checking classloading for 423277343d719e8aca173d7993f485d6
2024-12-11T16:26:24,162 INFO  [StoreOpener-423277343d719e8aca173d7993f485d6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 423277343d719e8aca173d7993f485d6 
2024-12-11T16:26:24,163 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40919 {}] regionserver.HRegion(8855): Flush requested on 1588230740
2024-12-11T16:26:24,163 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all.
2024-12-11T16:26:24,163 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=5.15 KB heapSize=9 KB
2024-12-11T16:26:24,163 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure f782a65af6e9ff7c34f274c17ab783a9, server=7446a65196a1,40919,1733934367992 because future has completed
2024-12-11T16:26:24,163 INFO  [StoreOpener-423277343d719e8aca173d7993f485d6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 423277343d719e8aca173d7993f485d6 columnFamilyName info
2024-12-11T16:26:24,163 DEBUG [StoreOpener-423277343d719e8aca173d7993f485d6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:26:24,167 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11
2024-12-11T16:26:24,167 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure f782a65af6e9ff7c34f274c17ab783a9, server=7446a65196a1,40919,1733934367992 in 204 msec
2024-12-11T16:26:24,169 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=f782a65af6e9ff7c34f274c17ab783a9, ASSIGN in 367 msec
2024-12-11T16:26:24,174 DEBUG [StoreOpener-423277343d719e8aca173d7993f485d6-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/423277343d719e8aca173d7993f485d6/info/72b4c685a5da4939a2dce6f5710cfe3e.c898cf8c864a19c76dae1a3f2f276629->hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/72b4c685a5da4939a2dce6f5710cfe3e-bottom
2024-12-11T16:26:24,174 INFO  [StoreOpener-423277343d719e8aca173d7993f485d6-1 {}] regionserver.HStore(327): Store=423277343d719e8aca173d7993f485d6/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-11T16:26:24,174 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1038): replaying wal for 423277343d719e8aca173d7993f485d6
2024-12-11T16:26:24,175 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/423277343d719e8aca173d7993f485d6
2024-12-11T16:26:24,176 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/423277343d719e8aca173d7993f485d6
2024-12-11T16:26:24,176 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1048): stopping wal replay for 423277343d719e8aca173d7993f485d6
2024-12-11T16:26:24,176 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1060): Cleaning up temporary data for 423277343d719e8aca173d7993f485d6
2024-12-11T16:26:24,178 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1093): writing seq id for 423277343d719e8aca173d7993f485d6
2024-12-11T16:26:24,178 INFO  [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1114): Opened 423277343d719e8aca173d7993f485d6; next sequenceid=131; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=744659, jitterRate=-0.053117379546165466}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-11T16:26:24,179 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 423277343d719e8aca173d7993f485d6
2024-12-11T16:26:24,179 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1006): Region open journal for 423277343d719e8aca173d7993f485d6:
Running coprocessor pre-open hook at 1733934384161Writing region info on filesystem at 1733934384161Initializing all the Stores at 1733934384162 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733934384162Cleaning up temporary data from old regions at 1733934384176 (+14 ms)Running coprocessor post-open hooks at 1733934384179 (+3 ms)Region opened successfully at 1733934384179
2024-12-11T16:26:24,179 INFO  [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,,1733934383447.423277343d719e8aca173d7993f485d6., pid=13, masterSystemTime=1733934384113
2024-12-11T16:26:24,179 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.CompactSplit(403): Add compact mark for store 423277343d719e8aca173d7993f485d6:info, priority=-2147483648, current under compaction store size is 2
2024-12-11T16:26:24,180 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-11T16:26:24,180 DEBUG [RS:0;7446a65196a1:40919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking
2024-12-11T16:26:24,180 INFO  [RS:0;7446a65196a1:40919-longCompactions-0 {}] regionserver.HStore(1527): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,,1733934383447.423277343d719e8aca173d7993f485d6.
2024-12-11T16:26:24,180 DEBUG [RS:0;7446a65196a1:40919-longCompactions-0 {}] regionserver.HStore(1541): 423277343d719e8aca173d7993f485d6/info is initiating minor compaction (all files)
2024-12-11T16:26:24,180 INFO  [RS:0;7446a65196a1:40919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 423277343d719e8aca173d7993f485d6/info in TestLogRolling-testLogRolling,,1733934383447.423277343d719e8aca173d7993f485d6.
2024-12-11T16:26:24,181 INFO  [RS:0;7446a65196a1:40919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/423277343d719e8aca173d7993f485d6/info/72b4c685a5da4939a2dce6f5710cfe3e.c898cf8c864a19c76dae1a3f2f276629->hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/72b4c685a5da4939a2dce6f5710cfe3e-bottom] into tmpdir=hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/423277343d719e8aca173d7993f485d6/.tmp, totalSize=87.4 K
2024-12-11T16:26:24,182 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/hbase/meta/1588230740/.tmp/info/dc66756c5f80483d84784b15239a630b is 193, key is TestLogRolling-testLogRolling,row0062,1733934383447.f782a65af6e9ff7c34f274c17ab783a9./info:regioninfo/1733934384161/Put/seqid=0
2024-12-11T16:26:24,182 DEBUG [RS:0;7446a65196a1:40919-longCompactions-0 {}] compactions.Compactor(225): Compacting 72b4c685a5da4939a2dce6f5710cfe3e.c898cf8c864a19c76dae1a3f2f276629, keycount=39, bloomtype=ROW, size=87.4 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1733934379220
2024-12-11T16:26:24,182 DEBUG [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,,1733934383447.423277343d719e8aca173d7993f485d6.
2024-12-11T16:26:24,182 INFO  [RS_OPEN_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,,1733934383447.423277343d719e8aca173d7993f485d6.
2024-12-11T16:26:24,183 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=423277343d719e8aca173d7993f485d6, regionState=OPEN, openSeqNum=131, regionLocation=7446a65196a1,40919,1733934367992
2024-12-11T16:26:24,185 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 423277343d719e8aca173d7993f485d6, server=7446a65196a1,40919,1733934367992 because future has completed
2024-12-11T16:26:24,188 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42195 is added to blk_1073741853_1029 (size=9882)
2024-12-11T16:26:24,188 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45613 is added to blk_1073741853_1029 (size=9882)
2024-12-11T16:26:24,189 INFO  [RS:0;7446a65196a1:40919-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f782a65af6e9ff7c34f274c17ab783a9#info#compaction#68 average throughput is 17.96 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-11T16:26:24,189 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/.tmp/info/ada858f57f7e4b00ba2ee77a52c2cd23 is 1080, key is row0062/info:/1733934381351/Put/seqid=0
2024-12-11T16:26:24,190 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.95 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/hbase/meta/1588230740/.tmp/info/dc66756c5f80483d84784b15239a630b
2024-12-11T16:26:24,191 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=13, resume processing ppid=10
2024-12-11T16:26:24,191 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 423277343d719e8aca173d7993f485d6, server=7446a65196a1,40919,1733934367992 in 226 msec
2024-12-11T16:26:24,193 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=7
2024-12-11T16:26:24,193 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=423277343d719e8aca173d7993f485d6, ASSIGN in 391 msec
2024-12-11T16:26:24,194 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=c898cf8c864a19c76dae1a3f2f276629, daughterA=423277343d719e8aca173d7993f485d6, daughterB=f782a65af6e9ff7c34f274c17ab783a9 in 746 msec
2024-12-11T16:26:24,202 INFO  [RS:0;7446a65196a1:40919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 423277343d719e8aca173d7993f485d6#info#compaction#69 average throughput is 31.30 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-11T16:26:24,202 DEBUG [RS:0;7446a65196a1:40919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/423277343d719e8aca173d7993f485d6/.tmp/info/a0da396dc5a749d58e2c26f4286f5ba0 is 1080, key is row0001/info:/1733934379220/Put/seqid=0
2024-12-11T16:26:24,208 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45613 is added to blk_1073741854_1030 (size=42984)
2024-12-11T16:26:24,208 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42195 is added to blk_1073741854_1030 (size=42984)
2024-12-11T16:26:24,214 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/hbase/meta/1588230740/.tmp/ns/54f7298bc04b4e58bd2b6c781100ddac is 43, key is default/ns:d/1733934369069/Put/seqid=0
2024-12-11T16:26:24,215 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/.tmp/info/ada858f57f7e4b00ba2ee77a52c2cd23 as hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/ada858f57f7e4b00ba2ee77a52c2cd23
2024-12-11T16:26:24,216 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45613 is added to blk_1073741855_1031 (size=70862)
2024-12-11T16:26:24,217 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42195 is added to blk_1073741855_1031 (size=70862)
2024-12-11T16:26:24,219 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42195 is added to blk_1073741856_1032 (size=5153)
2024-12-11T16:26:24,219 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45613 is added to blk_1073741856_1032 (size=5153)
2024-12-11T16:26:24,219 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/hbase/meta/1588230740/.tmp/ns/54f7298bc04b4e58bd2b6c781100ddac
2024-12-11T16:26:24,222 DEBUG [RS:0;7446a65196a1:40919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/423277343d719e8aca173d7993f485d6/.tmp/info/a0da396dc5a749d58e2c26f4286f5ba0 as hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/423277343d719e8aca173d7993f485d6/info/a0da396dc5a749d58e2c26f4286f5ba0
2024-12-11T16:26:24,223 INFO  [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f782a65af6e9ff7c34f274c17ab783a9/info of f782a65af6e9ff7c34f274c17ab783a9 into ada858f57f7e4b00ba2ee77a52c2cd23(size=42.0 K), total size for store is 42.0 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-11T16:26:24,223 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f782a65af6e9ff7c34f274c17ab783a9:

2024-12-11T16:26:24,223 INFO  [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733934383447.f782a65af6e9ff7c34f274c17ab783a9., storeName=f782a65af6e9ff7c34f274c17ab783a9/info, priority=13, startTime=1733934384158; duration=0sec
2024-12-11T16:26:24,223 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-11T16:26:24,223 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f782a65af6e9ff7c34f274c17ab783a9:info
2024-12-11T16:26:24,228 INFO  [RS:0;7446a65196a1:40919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 1 (all) file(s) in 423277343d719e8aca173d7993f485d6/info of 423277343d719e8aca173d7993f485d6 into a0da396dc5a749d58e2c26f4286f5ba0(size=69.2 K), total size for store is 69.2 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-11T16:26:24,228 DEBUG [RS:0;7446a65196a1:40919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 423277343d719e8aca173d7993f485d6:

2024-12-11T16:26:24,228 INFO  [RS:0;7446a65196a1:40919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1733934383447.423277343d719e8aca173d7993f485d6., storeName=423277343d719e8aca173d7993f485d6/info, priority=15, startTime=1733934384179; duration=0sec
2024-12-11T16:26:24,228 DEBUG [RS:0;7446a65196a1:40919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-11T16:26:24,228 DEBUG [RS:0;7446a65196a1:40919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 423277343d719e8aca173d7993f485d6:info
2024-12-11T16:26:24,239 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/hbase/meta/1588230740/.tmp/table/997e0006ee9940a58be76d1a926aea34 is 65, key is TestLogRolling-testLogRolling/table:state/1733934369506/Put/seqid=0
2024-12-11T16:26:24,243 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42195 is added to blk_1073741857_1033 (size=5340)
2024-12-11T16:26:24,243 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45613 is added to blk_1073741857_1033 (size=5340)
2024-12-11T16:26:24,243 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=122 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/hbase/meta/1588230740/.tmp/table/997e0006ee9940a58be76d1a926aea34
2024-12-11T16:26:24,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/hbase/meta/1588230740/.tmp/info/dc66756c5f80483d84784b15239a630b as hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/hbase/meta/1588230740/info/dc66756c5f80483d84784b15239a630b
2024-12-11T16:26:24,253 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/hbase/meta/1588230740/info/dc66756c5f80483d84784b15239a630b, entries=30, sequenceid=17, filesize=9.7 K
2024-12-11T16:26:24,254 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/hbase/meta/1588230740/.tmp/ns/54f7298bc04b4e58bd2b6c781100ddac as hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/hbase/meta/1588230740/ns/54f7298bc04b4e58bd2b6c781100ddac
2024-12-11T16:26:24,259 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/hbase/meta/1588230740/ns/54f7298bc04b4e58bd2b6c781100ddac, entries=2, sequenceid=17, filesize=5.0 K
2024-12-11T16:26:24,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/hbase/meta/1588230740/.tmp/table/997e0006ee9940a58be76d1a926aea34 as hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/hbase/meta/1588230740/table/997e0006ee9940a58be76d1a926aea34
2024-12-11T16:26:24,265 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/hbase/meta/1588230740/table/997e0006ee9940a58be76d1a926aea34, entries=2, sequenceid=17, filesize=5.2 K
2024-12-11T16:26:24,266 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~5.15 KB/5269, heapSize ~8.70 KB/8912, currentSize=670 B/670 for 1588230740 in 103ms, sequenceid=17, compaction requested=false
2024-12-11T16:26:24,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1588230740:

2024-12-11T16:26:24,390 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:24,395 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:25,391 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:25,396 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:25,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40919 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:35506 deadline: 1733934395432, exception=org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1733934369132.c898cf8c864a19c76dae1a3f2f276629. is not online on 7446a65196a1,40919,1733934367992
2024-12-11T16:26:25,441 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestLogRolling-testLogRolling,,1733934369132.c898cf8c864a19c76dae1a3f2f276629., hostname=7446a65196a1,40919,1733934367992, seqNum=2 , the old value is region=TestLogRolling-testLogRolling,,1733934369132.c898cf8c864a19c76dae1a3f2f276629., hostname=7446a65196a1,40919,1733934367992, seqNum=2, error=org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1733934369132.c898cf8c864a19c76dae1a3f2f276629. is not online on 7446a65196a1,40919,1733934367992
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186)
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2943)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-12-11T16:26:25,442 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestLogRolling-testLogRolling,,1733934369132.c898cf8c864a19c76dae1a3f2f276629., hostname=7446a65196a1,40919,1733934367992, seqNum=2 is org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1733934369132.c898cf8c864a19c76dae1a3f2f276629. is not online on 7446a65196a1,40919,1733934367992
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186)
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2943)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-12-11T16:26:25,442 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=TestLogRolling-testLogRolling,,1733934369132.c898cf8c864a19c76dae1a3f2f276629., hostname=7446a65196a1,40919,1733934367992, seqNum=2 from cache
2024-12-11T16:26:26,392 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:26,396 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:27,393 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:27,397 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:28,394 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:28,398 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:28,700 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:26:28,700 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:26:28,701 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:26:28,701 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:26:28,701 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:26:28,701 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:26:28,701 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:26:28,702 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:26:28,720 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:26:28,721 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:26:28,721 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:26:28,721 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:26:28,721 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:26:28,722 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:26:28,725 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:26:28,726 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:26:28,726 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:26:28,728 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:26:29,235 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-11T16:26:29,237 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:26:29,237 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:26:29,237 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:26:29,238 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:26:29,238 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:26:29,238 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:26:29,239 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:26:29,239 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:26:29,261 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:26:29,262 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:26:29,262 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:26:29,262 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:26:29,262 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:26:29,263 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:26:29,266 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:26:29,266 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:26:29,266 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:26:29,268 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:26:29,395 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:29,398 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:30,396 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:30,399 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:31,397 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:31,400 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:32,398 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:32,400 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:33,399 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:33,401 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:34,399 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:34,402 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:35,400 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:35,402 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:35,466 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRolling', row='row0097', locateType=CURRENT is [region=TestLogRolling-testLogRolling,row0062,1733934383447.f782a65af6e9ff7c34f274c17ab783a9., hostname=7446a65196a1,40919,1733934367992, seqNum=131]
2024-12-11T16:26:35,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40919 {}] regionserver.HRegion(8855): Flush requested on f782a65af6e9ff7c34f274c17ab783a9
2024-12-11T16:26:35,480 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f782a65af6e9ff7c34f274c17ab783a9 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-11T16:26:35,488 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/.tmp/info/2c3158beae2f433595c8d4603b055541 is 1080, key is row0097/info:/1733934395467/Put/seqid=0
2024-12-11T16:26:35,496 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45613 is added to blk_1073741858_1034 (size=12516)
2024-12-11T16:26:35,496 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42195 is added to blk_1073741858_1034 (size=12516)
2024-12-11T16:26:35,497 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/.tmp/info/2c3158beae2f433595c8d4603b055541
2024-12-11T16:26:35,504 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/.tmp/info/2c3158beae2f433595c8d4603b055541 as hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/2c3158beae2f433595c8d4603b055541
2024-12-11T16:26:35,509 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/2c3158beae2f433595c8d4603b055541, entries=7, sequenceid=141, filesize=12.2 K
2024-12-11T16:26:35,510 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=13.66 KB/13988 for f782a65af6e9ff7c34f274c17ab783a9 in 30ms, sequenceid=141, compaction requested=false
2024-12-11T16:26:35,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f782a65af6e9ff7c34f274c17ab783a9:

2024-12-11T16:26:35,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40919 {}] regionserver.HRegion(8855): Flush requested on f782a65af6e9ff7c34f274c17ab783a9
2024-12-11T16:26:35,512 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f782a65af6e9ff7c34f274c17ab783a9 1/1 column families, dataSize=14.71 KB heapSize=16 KB
2024-12-11T16:26:35,517 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/.tmp/info/d08b2b61b17e4b93b21817a73202cd49 is 1080, key is row0104/info:/1733934395481/Put/seqid=0
2024-12-11T16:26:35,523 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45613 is added to blk_1073741859_1035 (size=20078)
2024-12-11T16:26:35,523 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42195 is added to blk_1073741859_1035 (size=20078)
2024-12-11T16:26:35,523 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.71 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/.tmp/info/d08b2b61b17e4b93b21817a73202cd49
2024-12-11T16:26:35,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/.tmp/info/d08b2b61b17e4b93b21817a73202cd49 as hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/d08b2b61b17e4b93b21817a73202cd49
2024-12-11T16:26:35,536 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/d08b2b61b17e4b93b21817a73202cd49, entries=14, sequenceid=158, filesize=19.6 K
2024-12-11T16:26:35,537 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~14.71 KB/15064, heapSize ~15.98 KB/16368, currentSize=10.51 KB/10760 for f782a65af6e9ff7c34f274c17ab783a9 in 25ms, sequenceid=158, compaction requested=true
2024-12-11T16:26:35,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f782a65af6e9ff7c34f274c17ab783a9:

2024-12-11T16:26:35,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f782a65af6e9ff7c34f274c17ab783a9:info, priority=-2147483648, current under compaction store size is 1
2024-12-11T16:26:35,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-11T16:26:35,537 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-11T16:26:35,538 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 75578 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-11T16:26:35,538 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.HStore(1541): f782a65af6e9ff7c34f274c17ab783a9/info is initiating minor compaction (all files)
2024-12-11T16:26:35,538 INFO  [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f782a65af6e9ff7c34f274c17ab783a9/info in TestLogRolling-testLogRolling,row0062,1733934383447.f782a65af6e9ff7c34f274c17ab783a9.
2024-12-11T16:26:35,538 INFO  [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/ada858f57f7e4b00ba2ee77a52c2cd23, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/2c3158beae2f433595c8d4603b055541, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/d08b2b61b17e4b93b21817a73202cd49] into tmpdir=hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/.tmp, totalSize=73.8 K
2024-12-11T16:26:35,538 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] compactions.Compactor(225): Compacting ada858f57f7e4b00ba2ee77a52c2cd23, keycount=35, bloomtype=ROW, size=42.0 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1733934381351
2024-12-11T16:26:35,539 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c3158beae2f433595c8d4603b055541, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1733934395467
2024-12-11T16:26:35,539 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] compactions.Compactor(225): Compacting d08b2b61b17e4b93b21817a73202cd49, keycount=14, bloomtype=ROW, size=19.6 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1733934395481
2024-12-11T16:26:35,549 INFO  [RS:0;7446a65196a1:40919-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f782a65af6e9ff7c34f274c17ab783a9#info#compaction#74 average throughput is 57.46 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-11T16:26:35,550 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/.tmp/info/1f63d916ed674cfb8252846474023c3a is 1080, key is row0062/info:/1733934381351/Put/seqid=0
2024-12-11T16:26:35,554 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42195 is added to blk_1073741860_1036 (size=65792)
2024-12-11T16:26:35,554 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45613 is added to blk_1073741860_1036 (size=65792)
2024-12-11T16:26:35,559 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/.tmp/info/1f63d916ed674cfb8252846474023c3a as hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/1f63d916ed674cfb8252846474023c3a
2024-12-11T16:26:35,564 INFO  [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f782a65af6e9ff7c34f274c17ab783a9/info of f782a65af6e9ff7c34f274c17ab783a9 into 1f63d916ed674cfb8252846474023c3a(size=64.3 K), total size for store is 64.3 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-11T16:26:35,564 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f782a65af6e9ff7c34f274c17ab783a9:

2024-12-11T16:26:35,564 INFO  [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733934383447.f782a65af6e9ff7c34f274c17ab783a9., storeName=f782a65af6e9ff7c34f274c17ab783a9/info, priority=13, startTime=1733934395537; duration=0sec
2024-12-11T16:26:35,564 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-11T16:26:35,564 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f782a65af6e9ff7c34f274c17ab783a9:info
2024-12-11T16:26:36,401 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:36,403 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:37,402 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:37,404 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:37,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40919 {}] regionserver.HRegion(8855): Flush requested on f782a65af6e9ff7c34f274c17ab783a9
2024-12-11T16:26:37,539 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f782a65af6e9ff7c34f274c17ab783a9 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB
2024-12-11T16:26:37,543 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/.tmp/info/b2712445783c46f8bd89393421c01883 is 1080, key is row0118/info:/1733934395514/Put/seqid=0
2024-12-11T16:26:37,547 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42195 is added to blk_1073741861_1037 (size=17906)
2024-12-11T16:26:37,548 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45613 is added to blk_1073741861_1037 (size=17906)
2024-12-11T16:26:37,548 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/.tmp/info/b2712445783c46f8bd89393421c01883
2024-12-11T16:26:37,553 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/.tmp/info/b2712445783c46f8bd89393421c01883 as hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/b2712445783c46f8bd89393421c01883
2024-12-11T16:26:37,559 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/b2712445783c46f8bd89393421c01883, entries=12, sequenceid=174, filesize=17.5 K
2024-12-11T16:26:37,560 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=10.51 KB/10760 for f782a65af6e9ff7c34f274c17ab783a9 in 21ms, sequenceid=174, compaction requested=false
2024-12-11T16:26:37,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f782a65af6e9ff7c34f274c17ab783a9:

2024-12-11T16:26:37,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40919 {}] regionserver.HRegion(8855): Flush requested on f782a65af6e9ff7c34f274c17ab783a9
2024-12-11T16:26:37,562 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f782a65af6e9ff7c34f274c17ab783a9 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB
2024-12-11T16:26:37,566 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/.tmp/info/087de2627cec4be89db0fee245017548 is 1080, key is row0130/info:/1733934397540/Put/seqid=0
2024-12-11T16:26:37,586 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45613 is added to blk_1073741862_1038 (size=17906)
2024-12-11T16:26:37,586 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42195 is added to blk_1073741862_1038 (size=17906)
2024-12-11T16:26:37,587 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/.tmp/info/087de2627cec4be89db0fee245017548
2024-12-11T16:26:37,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/.tmp/info/087de2627cec4be89db0fee245017548 as hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/087de2627cec4be89db0fee245017548
2024-12-11T16:26:37,604 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit.
org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=f782a65af6e9ff7c34f274c17ab783a9, server=7446a65196a1,40919,1733934367992
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT]
2024-12-11T16:26:37,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40919 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:35506 deadline: 1733934407604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=f782a65af6e9ff7c34f274c17ab783a9, server=7446a65196a1,40919,1733934367992
2024-12-11T16:26:37,605 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestLogRolling-testLogRolling,row0062,1733934383447.f782a65af6e9ff7c34f274c17ab783a9., hostname=7446a65196a1,40919,1733934367992, seqNum=131 , the old value is region=TestLogRolling-testLogRolling,row0062,1733934383447.f782a65af6e9ff7c34f274c17ab783a9., hostname=7446a65196a1,40919,1733934367992, seqNum=131, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=f782a65af6e9ff7c34f274c17ab783a9, server=7446a65196a1,40919,1733934367992
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308)
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329)
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216)
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-12-11T16:26:37,605 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/087de2627cec4be89db0fee245017548, entries=12, sequenceid=189, filesize=17.5 K
2024-12-11T16:26:37,606 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestLogRolling-testLogRolling,row0062,1733934383447.f782a65af6e9ff7c34f274c17ab783a9., hostname=7446a65196a1,40919,1733934367992, seqNum=131 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=f782a65af6e9ff7c34f274c17ab783a9, server=7446a65196a1,40919,1733934367992
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308)
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329)
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216)
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-12-11T16:26:37,606 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestLogRolling-testLogRolling,row0062,1733934383447.f782a65af6e9ff7c34f274c17ab783a9., hostname=7446a65196a1,40919,1733934367992, seqNum=131 because the exception is null or not the one we care about
2024-12-11T16:26:37,606 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=17.86 KB/18292 for f782a65af6e9ff7c34f274c17ab783a9 in 44ms, sequenceid=189, compaction requested=true
2024-12-11T16:26:37,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f782a65af6e9ff7c34f274c17ab783a9:

2024-12-11T16:26:37,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f782a65af6e9ff7c34f274c17ab783a9:info, priority=-2147483648, current under compaction store size is 1
2024-12-11T16:26:37,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-11T16:26:37,607 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-11T16:26:37,608 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 101604 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-11T16:26:37,608 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.HStore(1541): f782a65af6e9ff7c34f274c17ab783a9/info is initiating minor compaction (all files)
2024-12-11T16:26:37,608 INFO  [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f782a65af6e9ff7c34f274c17ab783a9/info in TestLogRolling-testLogRolling,row0062,1733934383447.f782a65af6e9ff7c34f274c17ab783a9.
2024-12-11T16:26:37,608 INFO  [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/1f63d916ed674cfb8252846474023c3a, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/b2712445783c46f8bd89393421c01883, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/087de2627cec4be89db0fee245017548] into tmpdir=hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/.tmp, totalSize=99.2 K
2024-12-11T16:26:37,608 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f63d916ed674cfb8252846474023c3a, keycount=56, bloomtype=ROW, size=64.3 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1733934381351
2024-12-11T16:26:37,609 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2712445783c46f8bd89393421c01883, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1733934395514
2024-12-11T16:26:37,609 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 087de2627cec4be89db0fee245017548, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1733934397540
2024-12-11T16:26:37,621 INFO  [RS:0;7446a65196a1:40919-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f782a65af6e9ff7c34f274c17ab783a9#info#compaction#77 average throughput is 41.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-11T16:26:37,621 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/.tmp/info/d9da87a225824118b153715a9d729398 is 1080, key is row0062/info:/1733934381351/Put/seqid=0
2024-12-11T16:26:37,629 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45613 is added to blk_1073741863_1039 (size=91843)
2024-12-11T16:26:37,629 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42195 is added to blk_1073741863_1039 (size=91843)
2024-12-11T16:26:37,635 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/.tmp/info/d9da87a225824118b153715a9d729398 as hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/d9da87a225824118b153715a9d729398
2024-12-11T16:26:37,641 INFO  [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f782a65af6e9ff7c34f274c17ab783a9/info of f782a65af6e9ff7c34f274c17ab783a9 into d9da87a225824118b153715a9d729398(size=89.7 K), total size for store is 89.7 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-11T16:26:37,641 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f782a65af6e9ff7c34f274c17ab783a9:

2024-12-11T16:26:37,641 INFO  [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733934383447.f782a65af6e9ff7c34f274c17ab783a9., storeName=f782a65af6e9ff7c34f274c17ab783a9/info, priority=13, startTime=1733934397606; duration=0sec
2024-12-11T16:26:37,641 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-11T16:26:37,641 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f782a65af6e9ff7c34f274c17ab783a9:info
2024-12-11T16:26:37,804 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-11T16:26:38,404 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:38,405 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:39,404 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:39,405 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:40,405 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:40,405 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:41,406 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:41,406 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:42,407 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:42,407 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:43,407 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:43,407 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:44,408 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:44,408 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:45,409 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:45,409 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:46,410 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:46,410 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:47,410 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:47,410 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:47,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40919 {}] regionserver.HRegion(8855): Flush requested on f782a65af6e9ff7c34f274c17ab783a9
2024-12-11T16:26:47,651 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f782a65af6e9ff7c34f274c17ab783a9 1/1 column families, dataSize=18.91 KB heapSize=20.50 KB
2024-12-11T16:26:47,655 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/.tmp/info/6a4aa74b5c014cdaa5ebdab0819253e2 is 1080, key is row0142/info:/1733934397563/Put/seqid=0
2024-12-11T16:26:47,660 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45613 is added to blk_1073741864_1040 (size=24394)
2024-12-11T16:26:47,660 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42195 is added to blk_1073741864_1040 (size=24394)
2024-12-11T16:26:47,661 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=18.91 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/.tmp/info/6a4aa74b5c014cdaa5ebdab0819253e2
2024-12-11T16:26:47,667 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/.tmp/info/6a4aa74b5c014cdaa5ebdab0819253e2 as hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/6a4aa74b5c014cdaa5ebdab0819253e2
2024-12-11T16:26:47,671 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/6a4aa74b5c014cdaa5ebdab0819253e2, entries=18, sequenceid=211, filesize=23.8 K
2024-12-11T16:26:47,672 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~18.91 KB/19368, heapSize ~20.48 KB/20976, currentSize=1.05 KB/1076 for f782a65af6e9ff7c34f274c17ab783a9 in 21ms, sequenceid=211, compaction requested=false
2024-12-11T16:26:47,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f782a65af6e9ff7c34f274c17ab783a9:

2024-12-11T16:26:48,412 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:48,412 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:49,413 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:49,413 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:49,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40919 {}] regionserver.HRegion(8855): Flush requested on f782a65af6e9ff7c34f274c17ab783a9
2024-12-11T16:26:49,672 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f782a65af6e9ff7c34f274c17ab783a9 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-11T16:26:49,677 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/.tmp/info/f4bafc32361147b79fcfe31a7d3bf1c0 is 1080, key is row0160/info:/1733934407652/Put/seqid=0
2024-12-11T16:26:49,682 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45613 is added to blk_1073741865_1041 (size=12516)
2024-12-11T16:26:49,682 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42195 is added to blk_1073741865_1041 (size=12516)
2024-12-11T16:26:49,717 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit.
org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=f782a65af6e9ff7c34f274c17ab783a9, server=7446a65196a1,40919,1733934367992
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT]
2024-12-11T16:26:49,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40919 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:35506 deadline: 1733934419717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=f782a65af6e9ff7c34f274c17ab783a9, server=7446a65196a1,40919,1733934367992
2024-12-11T16:26:49,718 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestLogRolling-testLogRolling,row0062,1733934383447.f782a65af6e9ff7c34f274c17ab783a9., hostname=7446a65196a1,40919,1733934367992, seqNum=131 , the old value is region=TestLogRolling-testLogRolling,row0062,1733934383447.f782a65af6e9ff7c34f274c17ab783a9., hostname=7446a65196a1,40919,1733934367992, seqNum=131, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=f782a65af6e9ff7c34f274c17ab783a9, server=7446a65196a1,40919,1733934367992
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308)
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329)
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216)
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-12-11T16:26:49,718 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestLogRolling-testLogRolling,row0062,1733934383447.f782a65af6e9ff7c34f274c17ab783a9., hostname=7446a65196a1,40919,1733934367992, seqNum=131 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=f782a65af6e9ff7c34f274c17ab783a9, server=7446a65196a1,40919,1733934367992
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308)
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329)
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216)
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-12-11T16:26:49,718 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestLogRolling-testLogRolling,row0062,1733934383447.f782a65af6e9ff7c34f274c17ab783a9., hostname=7446a65196a1,40919,1733934367992, seqNum=131 because the exception is null or not the one we care about
2024-12-11T16:26:49,801 INFO  [master/7446a65196a1:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-12-11T16:26:49,802 INFO  [master/7446a65196a1:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-12-11T16:26:50,083 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/.tmp/info/f4bafc32361147b79fcfe31a7d3bf1c0
2024-12-11T16:26:50,093 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/.tmp/info/f4bafc32361147b79fcfe31a7d3bf1c0 as hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/f4bafc32361147b79fcfe31a7d3bf1c0
2024-12-11T16:26:50,098 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/f4bafc32361147b79fcfe31a7d3bf1c0, entries=7, sequenceid=221, filesize=12.2 K
2024-12-11T16:26:50,099 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=23.12 KB/23672 for f782a65af6e9ff7c34f274c17ab783a9 in 427ms, sequenceid=221, compaction requested=true
2024-12-11T16:26:50,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f782a65af6e9ff7c34f274c17ab783a9:

2024-12-11T16:26:50,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f782a65af6e9ff7c34f274c17ab783a9:info, priority=-2147483648, current under compaction store size is 1
2024-12-11T16:26:50,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-11T16:26:50,099 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-11T16:26:50,100 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 128753 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-11T16:26:50,100 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.HStore(1541): f782a65af6e9ff7c34f274c17ab783a9/info is initiating minor compaction (all files)
2024-12-11T16:26:50,100 INFO  [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f782a65af6e9ff7c34f274c17ab783a9/info in TestLogRolling-testLogRolling,row0062,1733934383447.f782a65af6e9ff7c34f274c17ab783a9.
2024-12-11T16:26:50,100 INFO  [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/d9da87a225824118b153715a9d729398, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/6a4aa74b5c014cdaa5ebdab0819253e2, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/f4bafc32361147b79fcfe31a7d3bf1c0] into tmpdir=hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/.tmp, totalSize=125.7 K
2024-12-11T16:26:50,101 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9da87a225824118b153715a9d729398, keycount=80, bloomtype=ROW, size=89.7 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1733934381351
2024-12-11T16:26:50,101 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a4aa74b5c014cdaa5ebdab0819253e2, keycount=18, bloomtype=ROW, size=23.8 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733934397563
2024-12-11T16:26:50,101 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4bafc32361147b79fcfe31a7d3bf1c0, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1733934407652
2024-12-11T16:26:50,111 INFO  [RS:0;7446a65196a1:40919-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f782a65af6e9ff7c34f274c17ab783a9#info#compaction#80 average throughput is 53.87 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-11T16:26:50,112 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/.tmp/info/45e058c16adb4dd5b4c53d8710f5eb0d is 1080, key is row0062/info:/1733934381351/Put/seqid=0
2024-12-11T16:26:50,118 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42195 is added to blk_1073741866_1042 (size=118899)
2024-12-11T16:26:50,118 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45613 is added to blk_1073741866_1042 (size=118899)
2024-12-11T16:26:50,124 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/.tmp/info/45e058c16adb4dd5b4c53d8710f5eb0d as hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/45e058c16adb4dd5b4c53d8710f5eb0d
2024-12-11T16:26:50,129 INFO  [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f782a65af6e9ff7c34f274c17ab783a9/info of f782a65af6e9ff7c34f274c17ab783a9 into 45e058c16adb4dd5b4c53d8710f5eb0d(size=116.1 K), total size for store is 116.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-11T16:26:50,129 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f782a65af6e9ff7c34f274c17ab783a9:

2024-12-11T16:26:50,129 INFO  [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733934383447.f782a65af6e9ff7c34f274c17ab783a9., storeName=f782a65af6e9ff7c34f274c17ab783a9/info, priority=13, startTime=1733934410099; duration=0sec
2024-12-11T16:26:50,129 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-11T16:26:50,129 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f782a65af6e9ff7c34f274c17ab783a9:info
2024-12-11T16:26:50,413 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:50,413 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:51,414 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:51,414 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:52,415 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:52,415 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:53,416 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:53,416 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:53,981 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 20375
2024-12-11T16:26:54,416 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:54,416 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:55,417 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:55,417 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:56,418 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:56,418 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:57,418 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:57,418 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:58,199 DEBUG [master/7446a65196a1:0:becomeActiveMaster-MemStoreChunkPool Statistics {}] regionserver.ChunkCreator$MemStoreChunkPool$StatisticsThread(417): data stats (chunk size=2097152): current pool size=3, created chunk count=9, reused chunk count=66, reuseRatio=88.00%
2024-12-11T16:26:58,199 DEBUG [master/7446a65196a1:0:becomeActiveMaster-MemStoreChunkPool Statistics {}] regionserver.ChunkCreator$MemStoreChunkPool$StatisticsThread(417): index stats (chunk size=209715): current pool size=0, created chunk count=0, reused chunk count=0, reuseRatio=0
2024-12-11T16:26:58,419 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:58,419 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:59,420 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:59,420 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:26:59,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40919 {}] regionserver.HRegion(8855): Flush requested on f782a65af6e9ff7c34f274c17ab783a9
2024-12-11T16:26:59,793 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f782a65af6e9ff7c34f274c17ab783a9 1/1 column families, dataSize=24.17 KB heapSize=26.13 KB
2024-12-11T16:26:59,801 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/.tmp/info/a48bb7795e814e19b636325459f33550 is 1080, key is row0167/info:/1733934409673/Put/seqid=0
2024-12-11T16:26:59,807 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42195 is added to blk_1073741867_1043 (size=29784)
2024-12-11T16:26:59,807 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45613 is added to blk_1073741867_1043 (size=29784)
2024-12-11T16:26:59,809 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.17 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/.tmp/info/a48bb7795e814e19b636325459f33550
2024-12-11T16:26:59,816 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/.tmp/info/a48bb7795e814e19b636325459f33550 as hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/a48bb7795e814e19b636325459f33550
2024-12-11T16:26:59,822 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/a48bb7795e814e19b636325459f33550, entries=23, sequenceid=248, filesize=29.1 K
2024-12-11T16:26:59,823 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~24.17 KB/24748, heapSize ~26.11 KB/26736, currentSize=3.15 KB/3228 for f782a65af6e9ff7c34f274c17ab783a9 in 30ms, sequenceid=248, compaction requested=false
2024-12-11T16:26:59,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f782a65af6e9ff7c34f274c17ab783a9:

2024-12-11T16:27:00,421 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:27:00,421 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:27:01,421 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:27:01,421 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:27:01,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40919 {}] regionserver.HRegion(8855): Flush requested on f782a65af6e9ff7c34f274c17ab783a9
2024-12-11T16:27:01,817 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f782a65af6e9ff7c34f274c17ab783a9 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-11T16:27:01,821 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/.tmp/info/c6f2ffef22c34b81ad4aa6dd9e4f6b3e is 1080, key is row0190/info:/1733934419796/Put/seqid=0
2024-12-11T16:27:01,826 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45613 is added to blk_1073741868_1044 (size=12518)
2024-12-11T16:27:01,826 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42195 is added to blk_1073741868_1044 (size=12518)
2024-12-11T16:27:01,827 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/.tmp/info/c6f2ffef22c34b81ad4aa6dd9e4f6b3e
2024-12-11T16:27:01,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/.tmp/info/c6f2ffef22c34b81ad4aa6dd9e4f6b3e as hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/c6f2ffef22c34b81ad4aa6dd9e4f6b3e
2024-12-11T16:27:01,837 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/c6f2ffef22c34b81ad4aa6dd9e4f6b3e, entries=7, sequenceid=258, filesize=12.2 K
2024-12-11T16:27:01,838 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=10.51 KB/10760 for f782a65af6e9ff7c34f274c17ab783a9 in 21ms, sequenceid=258, compaction requested=true
2024-12-11T16:27:01,838 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f782a65af6e9ff7c34f274c17ab783a9:

2024-12-11T16:27:01,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f782a65af6e9ff7c34f274c17ab783a9:info, priority=-2147483648, current under compaction store size is 1
2024-12-11T16:27:01,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-11T16:27:01,838 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-11T16:27:01,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40919 {}] regionserver.HRegion(8855): Flush requested on f782a65af6e9ff7c34f274c17ab783a9
2024-12-11T16:27:01,839 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f782a65af6e9ff7c34f274c17ab783a9 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB
2024-12-11T16:27:01,839 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 161201 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-11T16:27:01,839 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.HStore(1541): f782a65af6e9ff7c34f274c17ab783a9/info is initiating minor compaction (all files)
2024-12-11T16:27:01,839 INFO  [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f782a65af6e9ff7c34f274c17ab783a9/info in TestLogRolling-testLogRolling,row0062,1733934383447.f782a65af6e9ff7c34f274c17ab783a9.
2024-12-11T16:27:01,840 INFO  [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/45e058c16adb4dd5b4c53d8710f5eb0d, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/a48bb7795e814e19b636325459f33550, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/c6f2ffef22c34b81ad4aa6dd9e4f6b3e] into tmpdir=hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/.tmp, totalSize=157.4 K
2024-12-11T16:27:01,840 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45e058c16adb4dd5b4c53d8710f5eb0d, keycount=105, bloomtype=ROW, size=116.1 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1733934381351
2024-12-11T16:27:01,840 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] compactions.Compactor(225): Compacting a48bb7795e814e19b636325459f33550, keycount=23, bloomtype=ROW, size=29.1 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1733934409673
2024-12-11T16:27:01,841 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6f2ffef22c34b81ad4aa6dd9e4f6b3e, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1733934419796
2024-12-11T16:27:01,844 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/.tmp/info/bb6d6d0d099b4a549d088f949c32fa28 is 1080, key is row0197/info:/1733934421818/Put/seqid=0
2024-12-11T16:27:01,850 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45613 is added to blk_1073741869_1045 (size=16839)
2024-12-11T16:27:01,850 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42195 is added to blk_1073741869_1045 (size=16839)
2024-12-11T16:27:01,850 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/.tmp/info/bb6d6d0d099b4a549d088f949c32fa28
2024-12-11T16:27:01,856 INFO  [RS:0;7446a65196a1:40919-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f782a65af6e9ff7c34f274c17ab783a9#info#compaction#84 average throughput is 69.27 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-11T16:27:01,856 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/.tmp/info/bb6d6d0d099b4a549d088f949c32fa28 as hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/bb6d6d0d099b4a549d088f949c32fa28
2024-12-11T16:27:01,857 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/.tmp/info/6ecc9d518c2b48cfa63208f736f2ef76 is 1080, key is row0062/info:/1733934381351/Put/seqid=0
2024-12-11T16:27:01,861 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/bb6d6d0d099b4a549d088f949c32fa28, entries=11, sequenceid=272, filesize=16.4 K
2024-12-11T16:27:01,862 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=9.46 KB/9684 for f782a65af6e9ff7c34f274c17ab783a9 in 23ms, sequenceid=272, compaction requested=false
2024-12-11T16:27:01,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f782a65af6e9ff7c34f274c17ab783a9:

2024-12-11T16:27:01,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40919 {}] regionserver.HRegion(8855): Flush requested on f782a65af6e9ff7c34f274c17ab783a9
2024-12-11T16:27:01,863 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f782a65af6e9ff7c34f274c17ab783a9 1/1 column families, dataSize=10.51 KB heapSize=11.50 KB
2024-12-11T16:27:01,873 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/.tmp/info/0b2b39b16b4843448e952f2c77b9054c is 1080, key is row0208/info:/1733934421840/Put/seqid=0
2024-12-11T16:27:01,875 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45613 is added to blk_1073741870_1046 (size=151548)
2024-12-11T16:27:01,875 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42195 is added to blk_1073741870_1046 (size=151548)
2024-12-11T16:27:01,877 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42195 is added to blk_1073741871_1047 (size=15760)
2024-12-11T16:27:01,878 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45613 is added to blk_1073741871_1047 (size=15760)
2024-12-11T16:27:01,878 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.51 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/.tmp/info/0b2b39b16b4843448e952f2c77b9054c
2024-12-11T16:27:01,881 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/.tmp/info/6ecc9d518c2b48cfa63208f736f2ef76 as hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/6ecc9d518c2b48cfa63208f736f2ef76
2024-12-11T16:27:01,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/.tmp/info/0b2b39b16b4843448e952f2c77b9054c as hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/0b2b39b16b4843448e952f2c77b9054c
2024-12-11T16:27:01,887 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/0b2b39b16b4843448e952f2c77b9054c, entries=10, sequenceid=285, filesize=15.4 K
2024-12-11T16:27:01,888 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~10.51 KB/10760, heapSize ~11.48 KB/11760, currentSize=7.36 KB/7532 for f782a65af6e9ff7c34f274c17ab783a9 in 25ms, sequenceid=285, compaction requested=false
2024-12-11T16:27:01,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f782a65af6e9ff7c34f274c17ab783a9:

2024-12-11T16:27:01,888 INFO  [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f782a65af6e9ff7c34f274c17ab783a9/info of f782a65af6e9ff7c34f274c17ab783a9 into 6ecc9d518c2b48cfa63208f736f2ef76(size=148.0 K), total size for store is 179.8 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-11T16:27:01,888 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f782a65af6e9ff7c34f274c17ab783a9:

2024-12-11T16:27:01,888 INFO  [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733934383447.f782a65af6e9ff7c34f274c17ab783a9., storeName=f782a65af6e9ff7c34f274c17ab783a9/info, priority=13, startTime=1733934421838; duration=0sec
2024-12-11T16:27:01,888 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-11T16:27:01,888 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f782a65af6e9ff7c34f274c17ab783a9:info
2024-12-11T16:27:02,422 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:27:02,422 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:27:03,423 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:27:03,423 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:27:03,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40919 {}] regionserver.HRegion(8855): Flush requested on f782a65af6e9ff7c34f274c17ab783a9
2024-12-11T16:27:03,880 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f782a65af6e9ff7c34f274c17ab783a9 1/1 column families, dataSize=8.41 KB heapSize=9.25 KB
2024-12-11T16:27:03,888 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/.tmp/info/9121625b95664512aaef3d02866c795b is 1080, key is row0218/info:/1733934421864/Put/seqid=0
2024-12-11T16:27:03,893 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42195 is added to blk_1073741872_1048 (size=13602)
2024-12-11T16:27:03,893 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45613 is added to blk_1073741872_1048 (size=13602)
2024-12-11T16:27:03,893 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.41 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/.tmp/info/9121625b95664512aaef3d02866c795b
2024-12-11T16:27:03,899 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/.tmp/info/9121625b95664512aaef3d02866c795b as hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/9121625b95664512aaef3d02866c795b
2024-12-11T16:27:03,905 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/9121625b95664512aaef3d02866c795b, entries=8, sequenceid=297, filesize=13.3 K
2024-12-11T16:27:03,906 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~8.41 KB/8608, heapSize ~9.23 KB/9456, currentSize=11.56 KB/11836 for f782a65af6e9ff7c34f274c17ab783a9 in 26ms, sequenceid=297, compaction requested=true
2024-12-11T16:27:03,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f782a65af6e9ff7c34f274c17ab783a9:

2024-12-11T16:27:03,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40919 {}] regionserver.HRegion(8855): Flush requested on f782a65af6e9ff7c34f274c17ab783a9
2024-12-11T16:27:03,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f782a65af6e9ff7c34f274c17ab783a9:info, priority=-2147483648, current under compaction store size is 1
2024-12-11T16:27:03,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-11T16:27:03,906 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking
2024-12-11T16:27:03,906 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f782a65af6e9ff7c34f274c17ab783a9 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB
2024-12-11T16:27:03,907 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4  files of size 197749 starting at candidate #0 after considering 3 permutations with 3 in ratio
2024-12-11T16:27:03,907 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.HStore(1541): f782a65af6e9ff7c34f274c17ab783a9/info is initiating minor compaction (all files)
2024-12-11T16:27:03,907 INFO  [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f782a65af6e9ff7c34f274c17ab783a9/info in TestLogRolling-testLogRolling,row0062,1733934383447.f782a65af6e9ff7c34f274c17ab783a9.
2024-12-11T16:27:03,908 INFO  [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/6ecc9d518c2b48cfa63208f736f2ef76, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/bb6d6d0d099b4a549d088f949c32fa28, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/0b2b39b16b4843448e952f2c77b9054c, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/9121625b95664512aaef3d02866c795b] into tmpdir=hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/.tmp, totalSize=193.1 K
2024-12-11T16:27:03,908 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ecc9d518c2b48cfa63208f736f2ef76, keycount=135, bloomtype=ROW, size=148.0 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1733934381351
2024-12-11T16:27:03,908 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb6d6d0d099b4a549d088f949c32fa28, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1733934421818
2024-12-11T16:27:03,908 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b2b39b16b4843448e952f2c77b9054c, keycount=10, bloomtype=ROW, size=15.4 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1733934421840
2024-12-11T16:27:03,909 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9121625b95664512aaef3d02866c795b, keycount=8, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1733934421864
2024-12-11T16:27:03,910 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/.tmp/info/0fd08f2d4f8f41558d8d17a09a337a95 is 1080, key is row0226/info:/1733934423882/Put/seqid=0
2024-12-11T16:27:03,916 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42195 is added to blk_1073741873_1049 (size=17918)
2024-12-11T16:27:03,917 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45613 is added to blk_1073741873_1049 (size=17918)
2024-12-11T16:27:03,917 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/.tmp/info/0fd08f2d4f8f41558d8d17a09a337a95
2024-12-11T16:27:03,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/.tmp/info/0fd08f2d4f8f41558d8d17a09a337a95 as hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/0fd08f2d4f8f41558d8d17a09a337a95
2024-12-11T16:27:03,924 INFO  [RS:0;7446a65196a1:40919-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f782a65af6e9ff7c34f274c17ab783a9#info#compaction#88 average throughput is 56.10 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-11T16:27:03,925 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/.tmp/info/406b544452024db2aab0531e3efe4dbd is 1080, key is row0062/info:/1733934381351/Put/seqid=0
2024-12-11T16:27:03,928 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/0fd08f2d4f8f41558d8d17a09a337a95, entries=12, sequenceid=312, filesize=17.5 K
2024-12-11T16:27:03,929 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=11.56 KB/11836 for f782a65af6e9ff7c34f274c17ab783a9 in 23ms, sequenceid=312, compaction requested=false
2024-12-11T16:27:03,929 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f782a65af6e9ff7c34f274c17ab783a9:

2024-12-11T16:27:03,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40919 {}] regionserver.HRegion(8855): Flush requested on f782a65af6e9ff7c34f274c17ab783a9
2024-12-11T16:27:03,931 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f782a65af6e9ff7c34f274c17ab783a9 1/1 column families, dataSize=13.66 KB heapSize=14.88 KB
2024-12-11T16:27:03,932 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45613 is added to blk_1073741874_1050 (size=182983)
2024-12-11T16:27:03,932 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42195 is added to blk_1073741874_1050 (size=182983)
2024-12-11T16:27:03,935 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/.tmp/info/23a2f9bf0a03416391cd0ed8ceae0d21 is 1080, key is row0238/info:/1733934423907/Put/seqid=0
2024-12-11T16:27:03,938 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/.tmp/info/406b544452024db2aab0531e3efe4dbd as hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/406b544452024db2aab0531e3efe4dbd
2024-12-11T16:27:03,939 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42195 is added to blk_1073741875_1051 (size=19013)
2024-12-11T16:27:03,939 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45613 is added to blk_1073741875_1051 (size=19013)
2024-12-11T16:27:03,940 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.66 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/.tmp/info/23a2f9bf0a03416391cd0ed8ceae0d21
2024-12-11T16:27:03,945 INFO  [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f782a65af6e9ff7c34f274c17ab783a9/info of f782a65af6e9ff7c34f274c17ab783a9 into 406b544452024db2aab0531e3efe4dbd(size=178.7 K), total size for store is 196.2 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-11T16:27:03,945 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f782a65af6e9ff7c34f274c17ab783a9:

2024-12-11T16:27:03,945 INFO  [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733934383447.f782a65af6e9ff7c34f274c17ab783a9., storeName=f782a65af6e9ff7c34f274c17ab783a9/info, priority=12, startTime=1733934423906; duration=0sec
2024-12-11T16:27:03,945 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-11T16:27:03,945 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f782a65af6e9ff7c34f274c17ab783a9:info
2024-12-11T16:27:03,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/.tmp/info/23a2f9bf0a03416391cd0ed8ceae0d21 as hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/23a2f9bf0a03416391cd0ed8ceae0d21
2024-12-11T16:27:03,949 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/23a2f9bf0a03416391cd0ed8ceae0d21, entries=13, sequenceid=328, filesize=18.6 K
2024-12-11T16:27:03,950 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~13.66 KB/13988, heapSize ~14.86 KB/15216, currentSize=6.30 KB/6456 for f782a65af6e9ff7c34f274c17ab783a9 in 19ms, sequenceid=328, compaction requested=true
2024-12-11T16:27:03,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f782a65af6e9ff7c34f274c17ab783a9:

2024-12-11T16:27:03,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f782a65af6e9ff7c34f274c17ab783a9:info, priority=-2147483648, current under compaction store size is 1
2024-12-11T16:27:03,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-11T16:27:03,950 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-11T16:27:03,951 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 219914 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-11T16:27:03,951 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.HStore(1541): f782a65af6e9ff7c34f274c17ab783a9/info is initiating minor compaction (all files)
2024-12-11T16:27:03,951 INFO  [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f782a65af6e9ff7c34f274c17ab783a9/info in TestLogRolling-testLogRolling,row0062,1733934383447.f782a65af6e9ff7c34f274c17ab783a9.
2024-12-11T16:27:03,952 INFO  [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/406b544452024db2aab0531e3efe4dbd, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/0fd08f2d4f8f41558d8d17a09a337a95, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/23a2f9bf0a03416391cd0ed8ceae0d21] into tmpdir=hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/.tmp, totalSize=214.8 K
2024-12-11T16:27:03,952 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 406b544452024db2aab0531e3efe4dbd, keycount=164, bloomtype=ROW, size=178.7 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1733934381351
2024-12-11T16:27:03,952 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0fd08f2d4f8f41558d8d17a09a337a95, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1733934423882
2024-12-11T16:27:03,952 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23a2f9bf0a03416391cd0ed8ceae0d21, keycount=13, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1733934423907
2024-12-11T16:27:03,962 INFO  [RS:0;7446a65196a1:40919-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f782a65af6e9ff7c34f274c17ab783a9#info#compaction#90 average throughput is 64.65 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-11T16:27:03,962 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/.tmp/info/2f363967a3994d09a7b2a1f0706e4c01 is 1080, key is row0062/info:/1733934381351/Put/seqid=0
2024-12-11T16:27:03,964 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42195 is added to blk_1073741876_1052 (size=210137)
2024-12-11T16:27:03,964 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45613 is added to blk_1073741876_1052 (size=210137)
2024-12-11T16:27:03,969 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/.tmp/info/2f363967a3994d09a7b2a1f0706e4c01 as hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/2f363967a3994d09a7b2a1f0706e4c01
2024-12-11T16:27:03,974 INFO  [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f782a65af6e9ff7c34f274c17ab783a9/info of f782a65af6e9ff7c34f274c17ab783a9 into 2f363967a3994d09a7b2a1f0706e4c01(size=205.2 K), total size for store is 205.2 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-11T16:27:03,974 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f782a65af6e9ff7c34f274c17ab783a9:

2024-12-11T16:27:03,974 INFO  [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733934383447.f782a65af6e9ff7c34f274c17ab783a9., storeName=f782a65af6e9ff7c34f274c17ab783a9/info, priority=13, startTime=1733934423950; duration=0sec
2024-12-11T16:27:03,974 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-11T16:27:03,974 DEBUG [RS:0;7446a65196a1:40919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f782a65af6e9ff7c34f274c17ab783a9:info
2024-12-11T16:27:04,424 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:27:04,424 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:27:05,425 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:27:05,425 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:27:05,945 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(285): after writing there are 0 log files
2024-12-11T16:27:05,945 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7446a65196a1%2C40919%2C1733934367992.1733934425945
2024-12-11T16:27:05,958 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:27:05,958 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:27:05,958 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:27:05,958 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:27:05,958 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:27:05,958 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:27:05,959 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:27:05,959 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:27:05,982 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:27:05,982 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:27:05,982 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:27:05,982 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:27:05,982 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:27:05,982 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:27:05,986 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:27:05,986 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:27:05,986 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:27:05,986 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:27:05,986 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:27:05,986 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:27:05,986 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:27:05,986 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/WALs/7446a65196a1,40919,1733934367992/7446a65196a1%2C40919%2C1733934367992.1733934368603 with entries=315, filesize=309.36 KB; new WAL /user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/WALs/7446a65196a1,40919,1733934367992/7446a65196a1%2C40919%2C1733934367992.1733934425945
2024-12-11T16:27:05,986 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:27:05,988 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45613 is added to blk_1073741833_1009 (size=316797)
2024-12-11T16:27:05,988 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42195 is added to blk_1073741833_1009 (size=316797)
2024-12-11T16:27:05,988 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:27:05,995 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34135:34135),(127.0.0.1/127.0.0.1:35903:35903)]
2024-12-11T16:27:06,000 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing f782a65af6e9ff7c34f274c17ab783a9 1/1 column families, dataSize=6.30 KB heapSize=7 KB
2024-12-11T16:27:06,007 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/.tmp/info/05f817eb90e84fe980a9b00adbb99c32 is 1080, key is row0251/info:/1733934423933/Put/seqid=0
2024-12-11T16:27:06,024 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45613 is added to blk_1073741878_1054 (size=11436)
2024-12-11T16:27:06,025 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42195 is added to blk_1073741878_1054 (size=11436)
2024-12-11T16:27:06,025 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.30 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/.tmp/info/05f817eb90e84fe980a9b00adbb99c32
2024-12-11T16:27:06,037 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/.tmp/info/05f817eb90e84fe980a9b00adbb99c32 as hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/05f817eb90e84fe980a9b00adbb99c32
2024-12-11T16:27:06,046 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/05f817eb90e84fe980a9b00adbb99c32, entries=6, sequenceid=339, filesize=11.2 K
2024-12-11T16:27:06,048 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~6.30 KB/6456, heapSize ~6.98 KB/7152, currentSize=0 B/0 for f782a65af6e9ff7c34f274c17ab783a9 in 47ms, sequenceid=339, compaction requested=false
2024-12-11T16:27:06,048 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for f782a65af6e9ff7c34f274c17ab783a9:

2024-12-11T16:27:06,048 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=670 B heapSize=2.02 KB
2024-12-11T16:27:06,055 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/hbase/meta/1588230740/.tmp/info/113b8b5f67b044848096e5fef3223133 is 186, key is TestLogRolling-testLogRolling,,1733934383447.423277343d719e8aca173d7993f485d6./info:regioninfo/1733934384183/Put/seqid=0
2024-12-11T16:27:06,069 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42195 is added to blk_1073741879_1055 (size=6153)
2024-12-11T16:27:06,069 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45613 is added to blk_1073741879_1055 (size=6153)
2024-12-11T16:27:06,071 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=670 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/hbase/meta/1588230740/.tmp/info/113b8b5f67b044848096e5fef3223133
2024-12-11T16:27:06,080 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/hbase/meta/1588230740/.tmp/info/113b8b5f67b044848096e5fef3223133 as hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/hbase/meta/1588230740/info/113b8b5f67b044848096e5fef3223133
2024-12-11T16:27:06,092 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/hbase/meta/1588230740/info/113b8b5f67b044848096e5fef3223133, entries=5, sequenceid=21, filesize=6.0 K
2024-12-11T16:27:06,095 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~670 B/670, heapSize ~1.25 KB/1280, currentSize=0 B/0 for 1588230740 in 47ms, sequenceid=21, compaction requested=false
2024-12-11T16:27:06,096 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1588230740:

2024-12-11T16:27:06,096 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 423277343d719e8aca173d7993f485d6:

2024-12-11T16:27:06,096 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7446a65196a1%2C40919%2C1733934367992.1733934426096
2024-12-11T16:27:06,110 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:27:06,110 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:27:06,110 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:27:06,110 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:27:06,110 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:27:06,110 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/WALs/7446a65196a1,40919,1733934367992/7446a65196a1%2C40919%2C1733934367992.1733934425945 with entries=2, filesize=723 B; new WAL /user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/WALs/7446a65196a1,40919,1733934367992/7446a65196a1%2C40919%2C1733934367992.1733934426096
2024-12-11T16:27:06,111 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34135:34135),(127.0.0.1/127.0.0.1:35903:35903)]
2024-12-11T16:27:06,111 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/WALs/7446a65196a1,40919,1733934367992/7446a65196a1%2C40919%2C1733934367992.1733934425945 is not closed yet, will try archiving it next time
2024-12-11T16:27:06,114 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/WALs/7446a65196a1,40919,1733934367992/7446a65196a1%2C40919%2C1733934367992.1733934368603 to hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/oldWALs/7446a65196a1%2C40919%2C1733934367992.1733934368603
2024-12-11T16:27:06,115 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1])
2024-12-11T16:27:06,115 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42195 is added to blk_1073741877_1053 (size=731)
2024-12-11T16:27:06,117 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45613 is added to blk_1073741877_1053 (size=731)
2024-12-11T16:27:06,119 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/WALs/7446a65196a1,40919,1733934367992/7446a65196a1%2C40919%2C1733934367992.1733934425945 to hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/oldWALs/7446a65196a1%2C40919%2C1733934367992.1733934425945
2024-12-11T16:27:06,215 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-12-11T16:27:06,215 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-11T16:27:06,215 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-11T16:27:06,216 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T16:27:06,216 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T16:27:06,216 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-12-11T16:27:06,216 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-12-11T16:27:06,216 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1269016637, stopped=false
2024-12-11T16:27:06,216 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=7446a65196a1,39633,1733934367823
2024-12-11T16:27:06,246 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40919-0x10015c082dd0001, quorum=127.0.0.1:51025, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-11T16:27:06,246 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39633-0x10015c082dd0000, quorum=127.0.0.1:51025, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-11T16:27:06,246 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40919-0x10015c082dd0001, quorum=127.0.0.1:51025, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:27:06,246 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39633-0x10015c082dd0000, quorum=127.0.0.1:51025, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:27:06,246 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-11T16:27:06,246 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-11T16:27:06,247 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:39633-0x10015c082dd0000, quorum=127.0.0.1:51025, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-11T16:27:06,247 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-11T16:27:06,247 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T16:27:06,247 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '7446a65196a1,40919,1733934367992' *****
2024-12-11T16:27:06,247 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-12-11T16:27:06,247 INFO  [RS:0;7446a65196a1:40919 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-11T16:27:06,248 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-12-11T16:27:06,248 INFO  [RS:0;7446a65196a1:40919 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-11T16:27:06,248 INFO  [RS:0;7446a65196a1:40919 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-11T16:27:06,248 INFO  [RS:0;7446a65196a1:40919 {}] regionserver.HRegionServer(3091): Received CLOSE for f782a65af6e9ff7c34f274c17ab783a9
2024-12-11T16:27:06,248 INFO  [RS:0;7446a65196a1:40919 {}] regionserver.HRegionServer(3091): Received CLOSE for 423277343d719e8aca173d7993f485d6
2024-12-11T16:27:06,248 INFO  [RS:0;7446a65196a1:40919 {}] regionserver.HRegionServer(959): stopping server 7446a65196a1,40919,1733934367992
2024-12-11T16:27:06,248 INFO  [RS:0;7446a65196a1:40919 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-11T16:27:06,248 INFO  [RS:0;7446a65196a1:40919 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;7446a65196a1:40919.
2024-12-11T16:27:06,248 DEBUG [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f782a65af6e9ff7c34f274c17ab783a9, disabling compactions & flushes
2024-12-11T16:27:06,248 DEBUG [RS:0;7446a65196a1:40919 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-11T16:27:06,248 INFO  [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,row0062,1733934383447.f782a65af6e9ff7c34f274c17ab783a9.
2024-12-11T16:27:06,248 DEBUG [RS:0;7446a65196a1:40919 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T16:27:06,248 DEBUG [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,row0062,1733934383447.f782a65af6e9ff7c34f274c17ab783a9.
2024-12-11T16:27:06,248 DEBUG [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,row0062,1733934383447.f782a65af6e9ff7c34f274c17ab783a9. after waiting 0 ms
2024-12-11T16:27:06,248 DEBUG [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,row0062,1733934383447.f782a65af6e9ff7c34f274c17ab783a9.
2024-12-11T16:27:06,248 INFO  [RS:0;7446a65196a1:40919 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-11T16:27:06,248 INFO  [RS:0;7446a65196a1:40919 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-11T16:27:06,248 INFO  [RS:0;7446a65196a1:40919 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-11T16:27:06,248 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40919-0x10015c082dd0001, quorum=127.0.0.1:51025, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-11T16:27:06,248 INFO  [RS:0;7446a65196a1:40919 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-12-11T16:27:06,249 INFO  [RS:0;7446a65196a1:40919 {}] regionserver.HRegionServer(1321): Waiting on 3 regions to close
2024-12-11T16:27:06,249 DEBUG [RS:0;7446a65196a1:40919 {}] regionserver.HRegionServer(1325): Online Regions={f782a65af6e9ff7c34f274c17ab783a9=TestLogRolling-testLogRolling,row0062,1733934383447.f782a65af6e9ff7c34f274c17ab783a9., 1588230740=hbase:meta,,1.1588230740, 423277343d719e8aca173d7993f485d6=TestLogRolling-testLogRolling,,1733934383447.423277343d719e8aca173d7993f485d6.}
2024-12-11T16:27:06,249 DEBUG [RS:0;7446a65196a1:40919 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 423277343d719e8aca173d7993f485d6, f782a65af6e9ff7c34f274c17ab783a9
2024-12-11T16:27:06,249 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-11T16:27:06,249 INFO  [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-11T16:27:06,249 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-11T16:27:06,249 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-11T16:27:06,249 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-11T16:27:06,249 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733934383447.f782a65af6e9ff7c34f274c17ab783a9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/72b4c685a5da4939a2dce6f5710cfe3e.c898cf8c864a19c76dae1a3f2f276629->hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/72b4c685a5da4939a2dce6f5710cfe3e-top, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/TestLogRolling-testLogRolling=c898cf8c864a19c76dae1a3f2f276629-6cb30f3d036a4b5a9bd44a22492e4318, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/ada858f57f7e4b00ba2ee77a52c2cd23, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/TestLogRolling-testLogRolling=c898cf8c864a19c76dae1a3f2f276629-da090a8ba1e84fd3be7e4bfa6f99d5c8, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/2c3158beae2f433595c8d4603b055541, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/1f63d916ed674cfb8252846474023c3a, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/d08b2b61b17e4b93b21817a73202cd49, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/b2712445783c46f8bd89393421c01883, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/d9da87a225824118b153715a9d729398, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/087de2627cec4be89db0fee245017548, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/6a4aa74b5c014cdaa5ebdab0819253e2, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/45e058c16adb4dd5b4c53d8710f5eb0d, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/f4bafc32361147b79fcfe31a7d3bf1c0, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/a48bb7795e814e19b636325459f33550, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/6ecc9d518c2b48cfa63208f736f2ef76, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/c6f2ffef22c34b81ad4aa6dd9e4f6b3e, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/bb6d6d0d099b4a549d088f949c32fa28, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/0b2b39b16b4843448e952f2c77b9054c, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/406b544452024db2aab0531e3efe4dbd, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/9121625b95664512aaef3d02866c795b, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/0fd08f2d4f8f41558d8d17a09a337a95, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/23a2f9bf0a03416391cd0ed8ceae0d21] to archive
2024-12-11T16:27:06,250 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733934383447.f782a65af6e9ff7c34f274c17ab783a9.-1 {}] backup.HFileArchiver(363): Archiving compacted files.
2024-12-11T16:27:06,258 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/72b4c685a5da4939a2dce6f5710cfe3e.c898cf8c864a19c76dae1a3f2f276629 to hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/archive/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/72b4c685a5da4939a2dce6f5710cfe3e.c898cf8c864a19c76dae1a3f2f276629
2024-12-11T16:27:06,260 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/TestLogRolling-testLogRolling=c898cf8c864a19c76dae1a3f2f276629-6cb30f3d036a4b5a9bd44a22492e4318 to hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/archive/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/TestLogRolling-testLogRolling=c898cf8c864a19c76dae1a3f2f276629-6cb30f3d036a4b5a9bd44a22492e4318
2024-12-11T16:27:06,265 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/ada858f57f7e4b00ba2ee77a52c2cd23 to hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/archive/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/ada858f57f7e4b00ba2ee77a52c2cd23
2024-12-11T16:27:06,265 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/2c3158beae2f433595c8d4603b055541 to hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/archive/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/2c3158beae2f433595c8d4603b055541
2024-12-11T16:27:06,266 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/TestLogRolling-testLogRolling=c898cf8c864a19c76dae1a3f2f276629-da090a8ba1e84fd3be7e4bfa6f99d5c8 to hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/archive/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/TestLogRolling-testLogRolling=c898cf8c864a19c76dae1a3f2f276629-da090a8ba1e84fd3be7e4bfa6f99d5c8
2024-12-11T16:27:06,266 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1
2024-12-11T16:27:06,267 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/1f63d916ed674cfb8252846474023c3a to hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/archive/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/1f63d916ed674cfb8252846474023c3a
2024-12-11T16:27:06,268 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-11T16:27:06,268 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/d08b2b61b17e4b93b21817a73202cd49 to hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/archive/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/d08b2b61b17e4b93b21817a73202cd49
2024-12-11T16:27:06,268 INFO  [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-11T16:27:06,268 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733934426249Running coprocessor pre-close hooks at 1733934426249Disabling compacts and flushes for region at 1733934426249Disabling writes for close at 1733934426249Writing region close event to WAL at 1733934426260 (+11 ms)Running coprocessor post-close hooks at 1733934426267 (+7 ms)Closed at 1733934426268 (+1 ms)
2024-12-11T16:27:06,268 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/d9da87a225824118b153715a9d729398 to hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/archive/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/d9da87a225824118b153715a9d729398
2024-12-11T16:27:06,268 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-12-11T16:27:06,268 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/087de2627cec4be89db0fee245017548 to hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/archive/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/087de2627cec4be89db0fee245017548
2024-12-11T16:27:06,268 DEBUG [HFileArchiver-25 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/b2712445783c46f8bd89393421c01883 to hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/archive/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/b2712445783c46f8bd89393421c01883
2024-12-11T16:27:06,269 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/6a4aa74b5c014cdaa5ebdab0819253e2 to hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/archive/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/6a4aa74b5c014cdaa5ebdab0819253e2
2024-12-11T16:27:06,269 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/45e058c16adb4dd5b4c53d8710f5eb0d to hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/archive/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/45e058c16adb4dd5b4c53d8710f5eb0d
2024-12-11T16:27:06,269 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/f4bafc32361147b79fcfe31a7d3bf1c0 to hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/archive/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/f4bafc32361147b79fcfe31a7d3bf1c0
2024-12-11T16:27:06,269 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/a48bb7795e814e19b636325459f33550 to hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/archive/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/a48bb7795e814e19b636325459f33550
2024-12-11T16:27:06,270 DEBUG [HFileArchiver-25 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/6ecc9d518c2b48cfa63208f736f2ef76 to hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/archive/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/6ecc9d518c2b48cfa63208f736f2ef76
2024-12-11T16:27:06,270 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/bb6d6d0d099b4a549d088f949c32fa28 to hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/archive/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/bb6d6d0d099b4a549d088f949c32fa28
2024-12-11T16:27:06,270 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/c6f2ffef22c34b81ad4aa6dd9e4f6b3e to hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/archive/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/c6f2ffef22c34b81ad4aa6dd9e4f6b3e
2024-12-11T16:27:06,271 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/406b544452024db2aab0531e3efe4dbd to hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/archive/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/406b544452024db2aab0531e3efe4dbd
2024-12-11T16:27:06,271 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/0b2b39b16b4843448e952f2c77b9054c to hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/archive/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/0b2b39b16b4843448e952f2c77b9054c
2024-12-11T16:27:06,271 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/0fd08f2d4f8f41558d8d17a09a337a95 to hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/archive/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/0fd08f2d4f8f41558d8d17a09a337a95
2024-12-11T16:27:06,271 DEBUG [HFileArchiver-25 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/9121625b95664512aaef3d02866c795b to hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/archive/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/9121625b95664512aaef3d02866c795b
2024-12-11T16:27:06,271 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/23a2f9bf0a03416391cd0ed8ceae0d21 to hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/archive/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/info/23a2f9bf0a03416391cd0ed8ceae0d21
2024-12-11T16:27:06,272 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733934383447.f782a65af6e9ff7c34f274c17ab783a9.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=7446a65196a1:39633 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 16 more
2024-12-11T16:27:06,272 WARN  [StoreCloser-TestLogRolling-testLogRolling,row0062,1733934383447.f782a65af6e9ff7c34f274c17ab783a9.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [ada858f57f7e4b00ba2ee77a52c2cd23=42984, 2c3158beae2f433595c8d4603b055541=12516, 1f63d916ed674cfb8252846474023c3a=65792, d08b2b61b17e4b93b21817a73202cd49=20078, b2712445783c46f8bd89393421c01883=17906, d9da87a225824118b153715a9d729398=91843, 087de2627cec4be89db0fee245017548=17906, 6a4aa74b5c014cdaa5ebdab0819253e2=24394, 45e058c16adb4dd5b4c53d8710f5eb0d=118899, f4bafc32361147b79fcfe31a7d3bf1c0=12516, a48bb7795e814e19b636325459f33550=29784, 6ecc9d518c2b48cfa63208f736f2ef76=151548, c6f2ffef22c34b81ad4aa6dd9e4f6b3e=12518, bb6d6d0d099b4a549d088f949c32fa28=16839, 0b2b39b16b4843448e952f2c77b9054c=15760, 406b544452024db2aab0531e3efe4dbd=182983, 9121625b95664512aaef3d02866c795b=13602, 0fd08f2d4f8f41558d8d17a09a337a95=17918, 23a2f9bf0a03416391cd0ed8ceae0d21=19013]
2024-12-11T16:27:06,285 DEBUG [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/f782a65af6e9ff7c34f274c17ab783a9/recovered.edits/342.seqid, newMaxSeqId=342, maxSeqId=130
2024-12-11T16:27:06,286 INFO  [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,row0062,1733934383447.f782a65af6e9ff7c34f274c17ab783a9.
2024-12-11T16:27:06,286 DEBUG [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f782a65af6e9ff7c34f274c17ab783a9:
Waiting for close lock at 1733934426248Running coprocessor pre-close hooks at 1733934426248Disabling compacts and flushes for region at 1733934426248Disabling writes for close at 1733934426248Writing region close event to WAL at 1733934426282 (+34 ms)Running coprocessor post-close hooks at 1733934426286 (+4 ms)Closed at 1733934426286
2024-12-11T16:27:06,286 DEBUG [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,row0062,1733934383447.f782a65af6e9ff7c34f274c17ab783a9.
2024-12-11T16:27:06,286 DEBUG [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 423277343d719e8aca173d7993f485d6, disabling compactions & flushes
2024-12-11T16:27:06,286 INFO  [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1733934383447.423277343d719e8aca173d7993f485d6.
2024-12-11T16:27:06,286 DEBUG [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1733934383447.423277343d719e8aca173d7993f485d6.
2024-12-11T16:27:06,286 DEBUG [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1733934383447.423277343d719e8aca173d7993f485d6. after waiting 0 ms
2024-12-11T16:27:06,286 DEBUG [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1733934383447.423277343d719e8aca173d7993f485d6.
2024-12-11T16:27:06,287 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733934383447.423277343d719e8aca173d7993f485d6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/423277343d719e8aca173d7993f485d6/info/72b4c685a5da4939a2dce6f5710cfe3e.c898cf8c864a19c76dae1a3f2f276629->hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/c898cf8c864a19c76dae1a3f2f276629/info/72b4c685a5da4939a2dce6f5710cfe3e-bottom] to archive
2024-12-11T16:27:06,288 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733934383447.423277343d719e8aca173d7993f485d6.-1 {}] backup.HFileArchiver(363): Archiving compacted files.
2024-12-11T16:27:06,289 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/423277343d719e8aca173d7993f485d6/info/72b4c685a5da4939a2dce6f5710cfe3e.c898cf8c864a19c76dae1a3f2f276629 to hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/archive/data/default/TestLogRolling-testLogRolling/423277343d719e8aca173d7993f485d6/info/72b4c685a5da4939a2dce6f5710cfe3e.c898cf8c864a19c76dae1a3f2f276629
2024-12-11T16:27:06,289 WARN  [StoreCloser-TestLogRolling-testLogRolling,,1733934383447.423277343d719e8aca173d7993f485d6.-1 {}] regionserver.HStore(2414): Failed to report archival of files: []
2024-12-11T16:27:06,293 DEBUG [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/data/default/TestLogRolling-testLogRolling/423277343d719e8aca173d7993f485d6/recovered.edits/135.seqid, newMaxSeqId=135, maxSeqId=130
2024-12-11T16:27:06,294 INFO  [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1733934383447.423277343d719e8aca173d7993f485d6.
2024-12-11T16:27:06,294 DEBUG [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 423277343d719e8aca173d7993f485d6:
Waiting for close lock at 1733934426286Running coprocessor pre-close hooks at 1733934426286Disabling compacts and flushes for region at 1733934426286Disabling writes for close at 1733934426286Writing region close event to WAL at 1733934426290 (+4 ms)Running coprocessor post-close hooks at 1733934426294 (+4 ms)Closed at 1733934426294
2024-12-11T16:27:06,294 DEBUG [RS_CLOSE_REGION-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,,1733934383447.423277343d719e8aca173d7993f485d6.
2024-12-11T16:27:06,426 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:27:06,426 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:27:06,449 INFO  [RS:0;7446a65196a1:40919 {}] regionserver.HRegionServer(976): stopping server 7446a65196a1,40919,1733934367992; all regions closed.
2024-12-11T16:27:06,450 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:27:06,450 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:27:06,450 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:27:06,450 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:27:06,450 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:27:06,452 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45613 is added to blk_1073741834_1010 (size=8107)
2024-12-11T16:27:06,454 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42195 is added to blk_1073741834_1010 (size=8107)
2024-12-11T16:27:06,456 DEBUG [RS:0;7446a65196a1:40919 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/oldWALs
2024-12-11T16:27:06,456 INFO  [RS:0;7446a65196a1:40919 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 7446a65196a1%2C40919%2C1733934367992.meta:.meta(num 1733934368969)
2024-12-11T16:27:06,459 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:27:06,459 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:27:06,459 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:27:06,459 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:27:06,459 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:27:06,461 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42195 is added to blk_1073741880_1056 (size=780)
2024-12-11T16:27:06,463 DEBUG [RS:0;7446a65196a1:40919 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/oldWALs
2024-12-11T16:27:06,464 INFO  [RS:0;7446a65196a1:40919 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 7446a65196a1%2C40919%2C1733934367992:(num 1733934426096)
2024-12-11T16:27:06,464 DEBUG [RS:0;7446a65196a1:40919 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T16:27:06,464 INFO  [RS:0;7446a65196a1:40919 {}] regionserver.LeaseManager(133): Closed leases
2024-12-11T16:27:06,464 INFO  [RS:0;7446a65196a1:40919 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-11T16:27:06,464 INFO  [RS:0;7446a65196a1:40919 {}] hbase.ChoreService(370): Chore service for: regionserver/7446a65196a1:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown
2024-12-11T16:27:06,464 INFO  [RS:0;7446a65196a1:40919 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-11T16:27:06,464 INFO  [RS:0;7446a65196a1:40919 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40919
2024-12-11T16:27:06,467 INFO  [regionserver/7446a65196a1:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-11T16:27:06,468 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45613 is added to blk_1073741880_1056 (size=780)
2024-12-11T16:27:06,474 INFO  [regionserver/7446a65196a1:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-11T16:27:06,492 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40919-0x10015c082dd0001, quorum=127.0.0.1:51025, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/7446a65196a1,40919,1733934367992
2024-12-11T16:27:06,493 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39633-0x10015c082dd0000, quorum=127.0.0.1:51025, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-11T16:27:06,493 INFO  [RS:0;7446a65196a1:40919 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-11T16:27:06,493 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [7446a65196a1,40919,1733934367992]
2024-12-11T16:27:06,495 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-11T16:27:06,497 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:27:06,497 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:27:06,497 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:27:06,497 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:27:06,497 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:27:06,498 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:27:06,498 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:27:06,498 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:27:06,509 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/7446a65196a1,40919,1733934367992 already deleted, retry=false
2024-12-11T16:27:06,509 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 7446a65196a1,40919,1733934367992 expired; onlineServers=0
2024-12-11T16:27:06,509 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '7446a65196a1,39633,1733934367823' *****
2024-12-11T16:27:06,509 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-12-11T16:27:06,510 INFO  [M:0;7446a65196a1:39633 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-11T16:27:06,510 INFO  [M:0;7446a65196a1:39633 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-11T16:27:06,510 DEBUG [M:0;7446a65196a1:39633 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-12-11T16:27:06,510 DEBUG [M:0;7446a65196a1:39633 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-12-11T16:27:06,510 DEBUG [master/7446a65196a1:0:becomeActiveMaster-HFileCleaner.small.0-1733934368332 {}] cleaner.HFileCleaner(306): Exit Thread[master/7446a65196a1:0:becomeActiveMaster-HFileCleaner.small.0-1733934368332,5,FailOnTimeoutGroup]
2024-12-11T16:27:06,510 DEBUG [master/7446a65196a1:0:becomeActiveMaster-HFileCleaner.large.0-1733934368332 {}] cleaner.HFileCleaner(306): Exit Thread[master/7446a65196a1:0:becomeActiveMaster-HFileCleaner.large.0-1733934368332,5,FailOnTimeoutGroup]
2024-12-11T16:27:06,510 INFO  [M:0;7446a65196a1:39633 {}] hbase.ChoreService(370): Chore service for: master/7446a65196a1:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-12-11T16:27:06,510 INFO  [M:0;7446a65196a1:39633 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-11T16:27:06,510 DEBUG [M:0;7446a65196a1:39633 {}] master.HMaster(1795): Stopping service threads
2024-12-11T16:27:06,510 INFO  [M:0;7446a65196a1:39633 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-12-11T16:27:06,510 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-12-11T16:27:06,511 INFO  [M:0;7446a65196a1:39633 {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-11T16:27:06,511 INFO  [M:0;7446a65196a1:39633 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-12-11T16:27:06,511 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-12-11T16:27:06,521 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39633-0x10015c082dd0000, quorum=127.0.0.1:51025, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-12-11T16:27:06,521 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39633-0x10015c082dd0000, quorum=127.0.0.1:51025, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:27:06,529 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:27:06,529 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:27:06,530 DEBUG [M:0;7446a65196a1:39633 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/master already deleted, retry=false
2024-12-11T16:27:06,530 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:27:06,530 DEBUG [M:0;7446a65196a1:39633 {}] master.ActiveMasterManager(353): master:39633-0x10015c082dd0000, quorum=127.0.0.1:51025, baseZNode=/hbase Failed delete of our master address node; KeeperErrorCode = NoNode for /hbase/master
2024-12-11T16:27:06,530 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:27:06,530 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:27:06,531 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:27:06,531 INFO  [M:0;7446a65196a1:39633 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/.lastflushedseqids
2024-12-11T16:27:06,536 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:27:06,537 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:27:06,537 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:27:06,541 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:27:06,565 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45613 is added to blk_1073741881_1057 (size=228)
2024-12-11T16:27:06,565 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42195 is added to blk_1073741881_1057 (size=228)
2024-12-11T16:27:06,575 INFO  [M:0;7446a65196a1:39633 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-12-11T16:27:06,579 INFO  [M:0;7446a65196a1:39633 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-12-11T16:27:06,579 DEBUG [M:0;7446a65196a1:39633 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-11T16:27:06,579 INFO  [M:0;7446a65196a1:39633 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T16:27:06,579 DEBUG [M:0;7446a65196a1:39633 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T16:27:06,579 DEBUG [M:0;7446a65196a1:39633 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-11T16:27:06,579 DEBUG [M:0;7446a65196a1:39633 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T16:27:06,579 INFO  [M:0;7446a65196a1:39633 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=51.43 KB heapSize=63.38 KB
2024-12-11T16:27:06,603 INFO  [RS:0;7446a65196a1:40919 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-11T16:27:06,603 INFO  [RS:0;7446a65196a1:40919 {}] regionserver.HRegionServer(1031): Exiting; stopping=7446a65196a1,40919,1733934367992; zookeeper connection closed.
2024-12-11T16:27:06,607 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40919-0x10015c082dd0001, quorum=127.0.0.1:51025, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-11T16:27:06,607 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40919-0x10015c082dd0001, quorum=127.0.0.1:51025, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-11T16:27:06,609 DEBUG [M:0;7446a65196a1:39633 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/52a89b2f221741a99b6bb4fc7dd16db4 is 82, key is hbase:meta,,1/info:regioninfo/1733934368999/Put/seqid=0
2024-12-11T16:27:06,610 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@17ca67d3 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@17ca67d3
2024-12-11T16:27:06,610 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-12-11T16:27:06,628 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42195 is added to blk_1073741882_1058 (size=5672)
2024-12-11T16:27:06,628 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45613 is added to blk_1073741882_1058 (size=5672)
2024-12-11T16:27:06,629 INFO  [M:0;7446a65196a1:39633 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/52a89b2f221741a99b6bb4fc7dd16db4
2024-12-11T16:27:06,669 DEBUG [M:0;7446a65196a1:39633 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6721c9a626f0463682f8c8156e062f17 is 750, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733934369510/Put/seqid=0
2024-12-11T16:27:06,679 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42195 is added to blk_1073741883_1059 (size=7090)
2024-12-11T16:27:06,679 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45613 is added to blk_1073741883_1059 (size=7090)
2024-12-11T16:27:06,680 INFO  [M:0;7446a65196a1:39633 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=50.83 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6721c9a626f0463682f8c8156e062f17
2024-12-11T16:27:06,688 INFO  [M:0;7446a65196a1:39633 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 6721c9a626f0463682f8c8156e062f17
2024-12-11T16:27:06,705 DEBUG [M:0;7446a65196a1:39633 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9dc84545b5a444bfa02f882eb5a49cae is 69, key is 7446a65196a1,40919,1733934367992/rs:state/1733934368456/Put/seqid=0
2024-12-11T16:27:06,728 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42195 is added to blk_1073741884_1060 (size=5156)
2024-12-11T16:27:06,728 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45613 is added to blk_1073741884_1060 (size=5156)
2024-12-11T16:27:06,729 INFO  [M:0;7446a65196a1:39633 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9dc84545b5a444bfa02f882eb5a49cae
2024-12-11T16:27:06,754 DEBUG [M:0;7446a65196a1:39633 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/bfb650808df44ceaa6fe6921c7b714bc is 52, key is load_balancer_on/state:d/1733934369127/Put/seqid=0
2024-12-11T16:27:06,788 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42195 is added to blk_1073741885_1061 (size=5056)
2024-12-11T16:27:06,789 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45613 is added to blk_1073741885_1061 (size=5056)
2024-12-11T16:27:06,795 INFO  [M:0;7446a65196a1:39633 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/bfb650808df44ceaa6fe6921c7b714bc
2024-12-11T16:27:06,803 DEBUG [M:0;7446a65196a1:39633 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/52a89b2f221741a99b6bb4fc7dd16db4 as hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/52a89b2f221741a99b6bb4fc7dd16db4
2024-12-11T16:27:06,821 INFO  [M:0;7446a65196a1:39633 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/52a89b2f221741a99b6bb4fc7dd16db4, entries=8, sequenceid=125, filesize=5.5 K
2024-12-11T16:27:06,826 DEBUG [M:0;7446a65196a1:39633 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6721c9a626f0463682f8c8156e062f17 as hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/6721c9a626f0463682f8c8156e062f17
2024-12-11T16:27:06,846 INFO  [M:0;7446a65196a1:39633 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 6721c9a626f0463682f8c8156e062f17
2024-12-11T16:27:06,846 INFO  [M:0;7446a65196a1:39633 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/6721c9a626f0463682f8c8156e062f17, entries=13, sequenceid=125, filesize=6.9 K
2024-12-11T16:27:06,851 DEBUG [M:0;7446a65196a1:39633 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9dc84545b5a444bfa02f882eb5a49cae as hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/9dc84545b5a444bfa02f882eb5a49cae
2024-12-11T16:27:06,865 INFO  [M:0;7446a65196a1:39633 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/9dc84545b5a444bfa02f882eb5a49cae, entries=1, sequenceid=125, filesize=5.0 K
2024-12-11T16:27:06,868 DEBUG [M:0;7446a65196a1:39633 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/bfb650808df44ceaa6fe6921c7b714bc as hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/bfb650808df44ceaa6fe6921c7b714bc
2024-12-11T16:27:06,916 INFO  [M:0;7446a65196a1:39633 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44955/user/jenkins/test-data/74f743f0-ea9b-6a58-005b-78e43689859f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/bfb650808df44ceaa6fe6921c7b714bc, entries=1, sequenceid=125, filesize=4.9 K
2024-12-11T16:27:06,931 INFO  [M:0;7446a65196a1:39633 {}] regionserver.HRegion(3140): Finished flush of dataSize ~51.43 KB/52663, heapSize ~63.32 KB/64840, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 352ms, sequenceid=125, compaction requested=false
2024-12-11T16:27:06,944 INFO  [M:0;7446a65196a1:39633 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T16:27:06,944 DEBUG [M:0;7446a65196a1:39633 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733934426579Disabling compacts and flushes for region at 1733934426579Disabling writes for close at 1733934426579Obtaining lock to block concurrent updates at 1733934426579Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733934426579Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=52663, getHeapSize=64840, getOffHeapSize=0, getCellsCount=148 at 1733934426580 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733934426587 (+7 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733934426587Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733934426609 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733934426609Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733934426636 (+27 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733934426668 (+32 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733934426669 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733934426688 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733934426704 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733934426704Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733934426734 (+30 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733934426753 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733934426753Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5064994b: reopening flushed file at 1733934426802 (+49 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1c5d7c33: reopening flushed file at 1733934426821 (+19 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@164bfb26: reopening flushed file at 1733934426846 (+25 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@bb733b7: reopening flushed file at 1733934426866 (+20 ms)Finished flush of dataSize ~51.43 KB/52663, heapSize ~63.32 KB/64840, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 352ms, sequenceid=125, compaction requested=false at 1733934426931 (+65 ms)Writing region close event to WAL at 1733934426943 (+12 ms)Closed at 1733934426943
2024-12-11T16:27:06,944 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:27:06,944 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:27:06,944 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:27:06,947 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:27:06,951 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:27:06,963 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42195 is added to blk_1073741830_1006 (size=61332)
2024-12-11T16:27:06,963 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45613 is added to blk_1073741830_1006 (size=61332)
2024-12-11T16:27:07,192 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-11T16:27:07,192 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-12-11T16:27:07,192 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-12-11T16:27:07,192 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling
2024-12-11T16:27:07,362 INFO  [M:0;7446a65196a1:39633 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-12-11T16:27:07,362 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-11T16:27:07,362 INFO  [M:0;7446a65196a1:39633 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39633
2024-12-11T16:27:07,362 INFO  [M:0;7446a65196a1:39633 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-11T16:27:07,427 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:27:07,427 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:27:07,484 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39633-0x10015c082dd0000, quorum=127.0.0.1:51025, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-11T16:27:07,484 INFO  [M:0;7446a65196a1:39633 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-11T16:27:07,484 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39633-0x10015c082dd0000, quorum=127.0.0.1:51025, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-11T16:27:07,487 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@24e33e74{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T16:27:07,488 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3e392bcf{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-11T16:27:07,488 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-11T16:27:07,488 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@24fba7ba{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-11T16:27:07,488 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4f9d87e6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91bb4f8c-44ab-4329-ecbc-8f1e37d48355/hadoop.log.dir/,STOPPED}
2024-12-11T16:27:07,489 WARN  [BP-470135049-172.17.0.2-1733934366087 heartbeating to localhost/127.0.0.1:44955 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-11T16:27:07,489 WARN  [BP-470135049-172.17.0.2-1733934366087 heartbeating to localhost/127.0.0.1:44955 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-470135049-172.17.0.2-1733934366087 (Datanode Uuid 09486516-a0d5-489e-92e7-11e53adfb6cd) service to localhost/127.0.0.1:44955
2024-12-11T16:27:07,490 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-11T16:27:07,490 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91bb4f8c-44ab-4329-ecbc-8f1e37d48355/cluster_8a707b7d-e55b-61a4-30e3-3320dbb3250b/data/data4/current/BP-470135049-172.17.0.2-1733934366087 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T16:27:07,490 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-11T16:27:07,490 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91bb4f8c-44ab-4329-ecbc-8f1e37d48355/cluster_8a707b7d-e55b-61a4-30e3-3320dbb3250b/data/data3/current/BP-470135049-172.17.0.2-1733934366087 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T16:27:07,490 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-11T16:27:07,492 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@58d3c043{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T16:27:07,493 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7b1dc3c7{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-11T16:27:07,493 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-11T16:27:07,493 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@57d7e59e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-11T16:27:07,493 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2cf5c358{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91bb4f8c-44ab-4329-ecbc-8f1e37d48355/hadoop.log.dir/,STOPPED}
2024-12-11T16:27:07,494 WARN  [BP-470135049-172.17.0.2-1733934366087 heartbeating to localhost/127.0.0.1:44955 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-11T16:27:07,494 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-11T16:27:07,494 WARN  [BP-470135049-172.17.0.2-1733934366087 heartbeating to localhost/127.0.0.1:44955 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-470135049-172.17.0.2-1733934366087 (Datanode Uuid 9a84d457-85df-4ef8-9704-f07ec07f9ac5) service to localhost/127.0.0.1:44955
2024-12-11T16:27:07,494 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-11T16:27:07,494 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91bb4f8c-44ab-4329-ecbc-8f1e37d48355/cluster_8a707b7d-e55b-61a4-30e3-3320dbb3250b/data/data1/current/BP-470135049-172.17.0.2-1733934366087 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T16:27:07,495 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-11T16:27:07,495 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91bb4f8c-44ab-4329-ecbc-8f1e37d48355/cluster_8a707b7d-e55b-61a4-30e3-3320dbb3250b/data/data2/current/BP-470135049-172.17.0.2-1733934366087 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T16:27:07,501 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@49453556{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-11T16:27:07,502 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2bc7bf21{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-11T16:27:07,502 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-11T16:27:07,502 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7abfc7d2{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-11T16:27:07,502 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@48c6491c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91bb4f8c-44ab-4329-ecbc-8f1e37d48355/hadoop.log.dir/,STOPPED}
2024-12-11T16:27:07,509 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-12-11T16:27:07,541 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-12-11T16:27:07,553 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRolling Thread=235 (was 206)
Potentially hanging thread: HMaster-EventLoopGroup-14-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-14-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-38-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-38-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-23
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.6@localhost:44955
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-24
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-19
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-40-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-41-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-38-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (548703531) connection to localhost/127.0.0.1:44955 from jenkins.hfs.6
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-25
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-41-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:44955
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-39-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (548703531) connection to localhost/127.0.0.1:44955 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: HMaster-EventLoopGroup-14-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-41-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-40-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-39-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-21
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-20
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-18
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-40-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:44955
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:44955
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-22
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (548703531) connection to localhost/127.0.0.1:44955 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-39-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:44955
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=515 (was 485) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=353 (was 417), ProcessCount=11 (was 11), AvailableMemoryMB=1376 (was 252) - AvailableMemoryMB LEAK? -
2024-12-11T16:27:07,564 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnNothingWritten Thread=235, OpenFileDescriptor=515, MaxFileDescriptor=1048576, SystemLoadAverage=353, ProcessCount=11, AvailableMemoryMB=1375
2024-12-11T16:27:07,564 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-12-11T16:27:07,564 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91bb4f8c-44ab-4329-ecbc-8f1e37d48355/hadoop.log.dir so I do NOT create it in target/test-data/81c691dc-3bb4-2c8e-f7a7-649598fcf539
2024-12-11T16:27:07,564 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91bb4f8c-44ab-4329-ecbc-8f1e37d48355/hadoop.tmp.dir so I do NOT create it in target/test-data/81c691dc-3bb4-2c8e-f7a7-649598fcf539
2024-12-11T16:27:07,564 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81c691dc-3bb4-2c8e-f7a7-649598fcf539/cluster_7a27fac0-c771-36a1-1040-de522363619d, deleteOnExit=true
2024-12-11T16:27:07,565 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-12-11T16:27:07,565 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81c691dc-3bb4-2c8e-f7a7-649598fcf539/test.cache.data in system properties and HBase conf
2024-12-11T16:27:07,565 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81c691dc-3bb4-2c8e-f7a7-649598fcf539/hadoop.tmp.dir in system properties and HBase conf
2024-12-11T16:27:07,565 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81c691dc-3bb4-2c8e-f7a7-649598fcf539/hadoop.log.dir in system properties and HBase conf
2024-12-11T16:27:07,565 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81c691dc-3bb4-2c8e-f7a7-649598fcf539/mapreduce.cluster.local.dir in system properties and HBase conf
2024-12-11T16:27:07,565 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81c691dc-3bb4-2c8e-f7a7-649598fcf539/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-12-11T16:27:07,565 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-12-11T16:27:07,565 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-11T16:27:07,566 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81c691dc-3bb4-2c8e-f7a7-649598fcf539/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-12-11T16:27:07,566 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81c691dc-3bb4-2c8e-f7a7-649598fcf539/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-12-11T16:27:07,566 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81c691dc-3bb4-2c8e-f7a7-649598fcf539/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-12-11T16:27:07,566 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81c691dc-3bb4-2c8e-f7a7-649598fcf539/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-11T16:27:07,566 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81c691dc-3bb4-2c8e-f7a7-649598fcf539/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-12-11T16:27:07,566 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81c691dc-3bb4-2c8e-f7a7-649598fcf539/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-12-11T16:27:07,566 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81c691dc-3bb4-2c8e-f7a7-649598fcf539/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-11T16:27:07,566 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81c691dc-3bb4-2c8e-f7a7-649598fcf539/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-11T16:27:07,566 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81c691dc-3bb4-2c8e-f7a7-649598fcf539/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-12-11T16:27:07,566 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81c691dc-3bb4-2c8e-f7a7-649598fcf539/nfs.dump.dir in system properties and HBase conf
2024-12-11T16:27:07,567 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81c691dc-3bb4-2c8e-f7a7-649598fcf539/java.io.tmpdir in system properties and HBase conf
2024-12-11T16:27:07,567 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81c691dc-3bb4-2c8e-f7a7-649598fcf539/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-11T16:27:07,567 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81c691dc-3bb4-2c8e-f7a7-649598fcf539/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-12-11T16:27:07,567 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81c691dc-3bb4-2c8e-f7a7-649598fcf539/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-12-11T16:27:07,585 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-11T16:27:07,875 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-11T16:27:07,880 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-11T16:27:07,895 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-11T16:27:07,895 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-11T16:27:07,895 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-11T16:27:07,899 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-11T16:27:07,903 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@417d4fc2{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81c691dc-3bb4-2c8e-f7a7-649598fcf539/hadoop.log.dir/,AVAILABLE}
2024-12-11T16:27:07,903 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@79a533ce{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-11T16:27:08,008 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@443fd1e4{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81c691dc-3bb4-2c8e-f7a7-649598fcf539/java.io.tmpdir/jetty-localhost-44541-hadoop-hdfs-3_4_1-tests_jar-_-any-4729778387261768939/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-11T16:27:08,009 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@bc56624{HTTP/1.1, (http/1.1)}{localhost:44541}
2024-12-11T16:27:08,009 INFO  [Time-limited test {}] server.Server(415): Started @318679ms
2024-12-11T16:27:08,021 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-11T16:27:08,211 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-11T16:27:08,213 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-11T16:27:08,214 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-11T16:27:08,214 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-11T16:27:08,214 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-11T16:27:08,214 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5ebcbe6a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81c691dc-3bb4-2c8e-f7a7-649598fcf539/hadoop.log.dir/,AVAILABLE}
2024-12-11T16:27:08,215 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3b90357{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-11T16:27:08,306 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4981e138{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81c691dc-3bb4-2c8e-f7a7-649598fcf539/java.io.tmpdir/jetty-localhost-40291-hadoop-hdfs-3_4_1-tests_jar-_-any-4092866948423664073/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T16:27:08,307 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1f07cd2f{HTTP/1.1, (http/1.1)}{localhost:40291}
2024-12-11T16:27:08,307 INFO  [Time-limited test {}] server.Server(415): Started @318977ms
2024-12-11T16:27:08,308 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-11T16:27:08,336 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-11T16:27:08,340 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-11T16:27:08,341 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-11T16:27:08,341 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-11T16:27:08,341 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-11T16:27:08,341 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5154bbd2{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81c691dc-3bb4-2c8e-f7a7-649598fcf539/hadoop.log.dir/,AVAILABLE}
2024-12-11T16:27:08,342 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6650dd0b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-11T16:27:08,427 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:27:08,427 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:27:08,434 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@46b56cc{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81c691dc-3bb4-2c8e-f7a7-649598fcf539/java.io.tmpdir/jetty-localhost-42275-hadoop-hdfs-3_4_1-tests_jar-_-any-482275280689793115/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T16:27:08,434 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@fbadfdb{HTTP/1.1, (http/1.1)}{localhost:42275}
2024-12-11T16:27:08,434 INFO  [Time-limited test {}] server.Server(415): Started @319104ms
2024-12-11T16:27:08,435 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-11T16:27:09,131 WARN  [Thread-2508 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81c691dc-3bb4-2c8e-f7a7-649598fcf539/cluster_7a27fac0-c771-36a1-1040-de522363619d/data/data1/current/BP-313031944-172.17.0.2-1733934427590/current, will proceed with Du for space computation calculation, 
2024-12-11T16:27:09,131 WARN  [Thread-2509 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81c691dc-3bb4-2c8e-f7a7-649598fcf539/cluster_7a27fac0-c771-36a1-1040-de522363619d/data/data2/current/BP-313031944-172.17.0.2-1733934427590/current, will proceed with Du for space computation calculation, 
2024-12-11T16:27:09,188 WARN  [Thread-2472 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-11T16:27:09,197 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf73763592d17fb18 with lease ID 0x19e3aa3917c746ba: Processing first storage report for DS-56d4ef6a-d613-4508-b827-ce1c2588ad75 from datanode DatanodeRegistration(127.0.0.1:43057, datanodeUuid=6b7dba04-6238-40ee-8627-a43ae0cf1e09, infoPort=40365, infoSecurePort=0, ipcPort=45117, storageInfo=lv=-57;cid=testClusterID;nsid=1919941580;c=1733934427590)
2024-12-11T16:27:09,197 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf73763592d17fb18 with lease ID 0x19e3aa3917c746ba: from storage DS-56d4ef6a-d613-4508-b827-ce1c2588ad75 node DatanodeRegistration(127.0.0.1:43057, datanodeUuid=6b7dba04-6238-40ee-8627-a43ae0cf1e09, infoPort=40365, infoSecurePort=0, ipcPort=45117, storageInfo=lv=-57;cid=testClusterID;nsid=1919941580;c=1733934427590), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-11T16:27:09,197 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf73763592d17fb18 with lease ID 0x19e3aa3917c746ba: Processing first storage report for DS-648ebaaf-be36-4170-82e8-5bb84907b9ca from datanode DatanodeRegistration(127.0.0.1:43057, datanodeUuid=6b7dba04-6238-40ee-8627-a43ae0cf1e09, infoPort=40365, infoSecurePort=0, ipcPort=45117, storageInfo=lv=-57;cid=testClusterID;nsid=1919941580;c=1733934427590)
2024-12-11T16:27:09,197 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf73763592d17fb18 with lease ID 0x19e3aa3917c746ba: from storage DS-648ebaaf-be36-4170-82e8-5bb84907b9ca node DatanodeRegistration(127.0.0.1:43057, datanodeUuid=6b7dba04-6238-40ee-8627-a43ae0cf1e09, infoPort=40365, infoSecurePort=0, ipcPort=45117, storageInfo=lv=-57;cid=testClusterID;nsid=1919941580;c=1733934427590), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-11T16:27:09,260 WARN  [Thread-2519 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81c691dc-3bb4-2c8e-f7a7-649598fcf539/cluster_7a27fac0-c771-36a1-1040-de522363619d/data/data3/current/BP-313031944-172.17.0.2-1733934427590/current, will proceed with Du for space computation calculation, 
2024-12-11T16:27:09,267 WARN  [Thread-2520 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81c691dc-3bb4-2c8e-f7a7-649598fcf539/cluster_7a27fac0-c771-36a1-1040-de522363619d/data/data4/current/BP-313031944-172.17.0.2-1733934427590/current, will proceed with Du for space computation calculation, 
2024-12-11T16:27:09,307 WARN  [Thread-2495 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-11T16:27:09,313 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xff169b64e9eb4fa5 with lease ID 0x19e3aa3917c746bb: Processing first storage report for DS-051a4fb7-8a6e-46e4-be7a-ee4f4e02e21d from datanode DatanodeRegistration(127.0.0.1:36765, datanodeUuid=09d0c22a-7611-4458-883f-f262917054ee, infoPort=42477, infoSecurePort=0, ipcPort=39811, storageInfo=lv=-57;cid=testClusterID;nsid=1919941580;c=1733934427590)
2024-12-11T16:27:09,313 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xff169b64e9eb4fa5 with lease ID 0x19e3aa3917c746bb: from storage DS-051a4fb7-8a6e-46e4-be7a-ee4f4e02e21d node DatanodeRegistration(127.0.0.1:36765, datanodeUuid=09d0c22a-7611-4458-883f-f262917054ee, infoPort=42477, infoSecurePort=0, ipcPort=39811, storageInfo=lv=-57;cid=testClusterID;nsid=1919941580;c=1733934427590), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-11T16:27:09,313 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xff169b64e9eb4fa5 with lease ID 0x19e3aa3917c746bb: Processing first storage report for DS-fbb102cd-364e-406f-9801-65da9b443e99 from datanode DatanodeRegistration(127.0.0.1:36765, datanodeUuid=09d0c22a-7611-4458-883f-f262917054ee, infoPort=42477, infoSecurePort=0, ipcPort=39811, storageInfo=lv=-57;cid=testClusterID;nsid=1919941580;c=1733934427590)
2024-12-11T16:27:09,313 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xff169b64e9eb4fa5 with lease ID 0x19e3aa3917c746bb: from storage DS-fbb102cd-364e-406f-9801-65da9b443e99 node DatanodeRegistration(127.0.0.1:36765, datanodeUuid=09d0c22a-7611-4458-883f-f262917054ee, infoPort=42477, infoSecurePort=0, ipcPort=39811, storageInfo=lv=-57;cid=testClusterID;nsid=1919941580;c=1733934427590), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-11T16:27:09,361 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81c691dc-3bb4-2c8e-f7a7-649598fcf539
2024-12-11T16:27:09,368 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81c691dc-3bb4-2c8e-f7a7-649598fcf539/cluster_7a27fac0-c771-36a1-1040-de522363619d/zookeeper_0, clientPort=62362, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81c691dc-3bb4-2c8e-f7a7-649598fcf539/cluster_7a27fac0-c771-36a1-1040-de522363619d/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81c691dc-3bb4-2c8e-f7a7-649598fcf539/cluster_7a27fac0-c771-36a1-1040-de522363619d/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-11T16:27:09,368 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=62362
2024-12-11T16:27:09,369 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T16:27:09,370 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T16:27:09,380 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43057 is added to blk_1073741825_1001 (size=7)
2024-12-11T16:27:09,380 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36765 is added to blk_1073741825_1001 (size=7)
2024-12-11T16:27:09,382 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:39461/user/jenkins/test-data/4e5db2d3-08f7-3881-3540-ed81e84fdce7 with version=8
2024-12-11T16:27:09,382 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:36921/user/jenkins/test-data/fad4d092-9325-4f94-a207-a5a8df36527c/hbase-staging
2024-12-11T16:27:09,385 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/7446a65196a1:0 server-side Connection retries=45
2024-12-11T16:27:09,385 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-11T16:27:09,385 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-11T16:27:09,385 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-11T16:27:09,385 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-11T16:27:09,385 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-11T16:27:09,385 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-12-11T16:27:09,386 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-11T16:27:09,386 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43423
2024-12-11T16:27:09,388 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43423 connecting to ZooKeeper ensemble=127.0.0.1:62362
2024-12-11T16:27:09,428 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:27:09,428 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:27:09,439 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:434230x0, quorum=127.0.0.1:62362, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-11T16:27:09,439 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43423-0x10015c173510000 connected
2024-12-11T16:27:09,515 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T16:27:09,517 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T16:27:09,519 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43423-0x10015c173510000, quorum=127.0.0.1:62362, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-11T16:27:09,519 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:39461/user/jenkins/test-data/4e5db2d3-08f7-3881-3540-ed81e84fdce7, hbase.cluster.distributed=false
2024-12-11T16:27:09,521 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43423-0x10015c173510000, quorum=127.0.0.1:62362, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-11T16:27:09,521 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43423
2024-12-11T16:27:09,522 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43423
2024-12-11T16:27:09,523 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43423
2024-12-11T16:27:09,525 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43423
2024-12-11T16:27:09,525 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43423
2024-12-11T16:27:09,538 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/7446a65196a1:0 server-side Connection retries=45
2024-12-11T16:27:09,539 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-11T16:27:09,539 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-11T16:27:09,539 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-11T16:27:09,539 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-11T16:27:09,539 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-11T16:27:09,539 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-11T16:27:09,539 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-11T16:27:09,539 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42551
2024-12-11T16:27:09,540 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42551 connecting to ZooKeeper ensemble=127.0.0.1:62362
2024-12-11T16:27:09,541 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T16:27:09,543 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T16:27:09,557 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:425510x0, quorum=127.0.0.1:62362, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-11T16:27:09,557 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42551-0x10015c173510001 connected
2024-12-11T16:27:09,557 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:425510x0, quorum=127.0.0.1:62362, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-11T16:27:09,558 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-11T16:27:09,558 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-11T16:27:09,559 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42551-0x10015c173510001, quorum=127.0.0.1:62362, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-11T16:27:09,560 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42551-0x10015c173510001, quorum=127.0.0.1:62362, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-11T16:27:09,560 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42551
2024-12-11T16:27:09,560 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42551
2024-12-11T16:27:09,560 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42551
2024-12-11T16:27:09,561 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42551
2024-12-11T16:27:09,561 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42551
2024-12-11T16:27:09,571 DEBUG [M:0;7446a65196a1:43423 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;7446a65196a1:43423
2024-12-11T16:27:09,571 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/7446a65196a1,43423,1733934429385
2024-12-11T16:27:09,587 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42551-0x10015c173510001, quorum=127.0.0.1:62362, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-11T16:27:09,587 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43423-0x10015c173510000, quorum=127.0.0.1:62362, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-11T16:27:09,588 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43423-0x10015c173510000, quorum=127.0.0.1:62362, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/7446a65196a1,43423,1733934429385
2024-12-11T16:27:09,625 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42551-0x10015c173510001, quorum=127.0.0.1:62362, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-11T16:27:09,625 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43423-0x10015c173510000, quorum=127.0.0.1:62362, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:27:09,625 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42551-0x10015c173510001, quorum=127.0.0.1:62362, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:27:09,625 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43423-0x10015c173510000, quorum=127.0.0.1:62362, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-11T16:27:09,626 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/7446a65196a1,43423,1733934429385 from backup master directory
2024-12-11T16:27:09,648 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43423-0x10015c173510000, quorum=127.0.0.1:62362, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/7446a65196a1,43423,1733934429385
2024-12-11T16:27:09,648 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42551-0x10015c173510001, quorum=127.0.0.1:62362, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-11T16:27:09,648 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43423-0x10015c173510000, quorum=127.0.0.1:62362, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-11T16:27:09,648 WARN  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-11T16:27:09,648 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=7446a65196a1,43423,1733934429385
2024-12-11T16:27:09,652 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:39461/user/jenkins/test-data/4e5db2d3-08f7-3881-3540-ed81e84fdce7/hbase.id] with ID: 3127422d-d9e7-4b79-bb99-dc40b8a7b862
2024-12-11T16:27:09,652 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:39461/user/jenkins/test-data/4e5db2d3-08f7-3881-3540-ed81e84fdce7/.tmp/hbase.id
2024-12-11T16:27:09,657 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36765 is added to blk_1073741826_1002 (size=42)
2024-12-11T16:27:09,657 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43057 is added to blk_1073741826_1002 (size=42)
2024-12-11T16:27:09,657 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:39461/user/jenkins/test-data/4e5db2d3-08f7-3881-3540-ed81e84fdce7/.tmp/hbase.id]:[hdfs://localhost:39461/user/jenkins/test-data/4e5db2d3-08f7-3881-3540-ed81e84fdce7/hbase.id]
2024-12-11T16:27:09,668 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T16:27:09,668 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-12-11T16:27:09,669 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms.
2024-12-11T16:27:09,696 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42551-0x10015c173510001, quorum=127.0.0.1:62362, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:27:09,696 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43423-0x10015c173510000, quorum=127.0.0.1:62362, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:27:09,709 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36765 is added to blk_1073741827_1003 (size=196)
2024-12-11T16:27:09,709 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43057 is added to blk_1073741827_1003 (size=196)
2024-12-11T16:27:09,710 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-11T16:27:09,711 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-11T16:27:09,711 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-11T16:27:09,725 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36765 is added to blk_1073741828_1004 (size=1189)
2024-12-11T16:27:09,725 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43057 is added to blk_1073741828_1004 (size=1189)
2024-12-11T16:27:09,726 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:39461/user/jenkins/test-data/4e5db2d3-08f7-3881-3540-ed81e84fdce7/MasterData/data/master/store
2024-12-11T16:27:09,743 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43057 is added to blk_1073741829_1005 (size=34)
2024-12-11T16:27:09,744 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36765 is added to blk_1073741829_1005 (size=34)
2024-12-11T16:27:09,745 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T16:27:09,745 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-11T16:27:09,745 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T16:27:09,746 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T16:27:09,746 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-11T16:27:09,746 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T16:27:09,746 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T16:27:09,746 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733934429745Disabling compacts and flushes for region at 1733934429745Disabling writes for close at 1733934429746 (+1 ms)Writing region close event to WAL at 1733934429746Closed at 1733934429746
2024-12-11T16:27:09,747 WARN  [master/7446a65196a1:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:39461/user/jenkins/test-data/4e5db2d3-08f7-3881-3540-ed81e84fdce7/MasterData/data/master/store/.initializing
2024-12-11T16:27:09,747 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:39461/user/jenkins/test-data/4e5db2d3-08f7-3881-3540-ed81e84fdce7/MasterData/WALs/7446a65196a1,43423,1733934429385
2024-12-11T16:27:09,750 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7446a65196a1%2C43423%2C1733934429385, suffix=, logDir=hdfs://localhost:39461/user/jenkins/test-data/4e5db2d3-08f7-3881-3540-ed81e84fdce7/MasterData/WALs/7446a65196a1,43423,1733934429385, archiveDir=hdfs://localhost:39461/user/jenkins/test-data/4e5db2d3-08f7-3881-3540-ed81e84fdce7/MasterData/oldWALs, maxLogs=10
2024-12-11T16:27:09,751 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7446a65196a1%2C43423%2C1733934429385.1733934429751
2024-12-11T16:27:09,758 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4e5db2d3-08f7-3881-3540-ed81e84fdce7/MasterData/WALs/7446a65196a1,43423,1733934429385/7446a65196a1%2C43423%2C1733934429385.1733934429751
2024-12-11T16:27:09,763 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42477:42477),(127.0.0.1/127.0.0.1:40365:40365)]
2024-12-11T16:27:09,771 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-11T16:27:09,771 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T16:27:09,771 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T16:27:09,771 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T16:27:09,775 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-11T16:27:09,777 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-11T16:27:09,777 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:27:09,778 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T16:27:09,778 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-11T16:27:09,779 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-11T16:27:09,779 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:27:09,780 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-11T16:27:09,780 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-11T16:27:09,781 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-11T16:27:09,781 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:27:09,782 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-11T16:27:09,782 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-11T16:27:09,783 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-11T16:27:09,783 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:27:09,783 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-11T16:27:09,783 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T16:27:09,784 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39461/user/jenkins/test-data/4e5db2d3-08f7-3881-3540-ed81e84fdce7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-11T16:27:09,784 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39461/user/jenkins/test-data/4e5db2d3-08f7-3881-3540-ed81e84fdce7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-11T16:27:09,785 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T16:27:09,785 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T16:27:09,786 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-11T16:27:09,787 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T16:27:09,792 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39461/user/jenkins/test-data/4e5db2d3-08f7-3881-3540-ed81e84fdce7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-11T16:27:09,792 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=780047, jitterRate=-0.00812007486820221}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-11T16:27:09,793 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1733934429771Initializing all the Stores at 1733934429772 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733934429772Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733934429775 (+3 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733934429775Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733934429775Cleaning up temporary data from old regions at 1733934429785 (+10 ms)Region opened successfully at 1733934429793 (+8 ms)
2024-12-11T16:27:09,799 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-11T16:27:09,802 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77a32523, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7446a65196a1/172.17.0.2:0
2024-12-11T16:27:09,803 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-12-11T16:27:09,803 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-11T16:27:09,803 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-11T16:27:09,803 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-11T16:27:09,804 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-12-11T16:27:09,804 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-12-11T16:27:09,804 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-11T16:27:09,806 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-11T16:27:09,807 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43423-0x10015c173510000, quorum=127.0.0.1:62362, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-11T16:27:09,835 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-12-11T16:27:09,835 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-11T16:27:09,836 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43423-0x10015c173510000, quorum=127.0.0.1:62362, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-11T16:27:09,877 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-12-11T16:27:09,877 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-11T16:27:09,878 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43423-0x10015c173510000, quorum=127.0.0.1:62362, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-11T16:27:09,895 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-12-11T16:27:09,897 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43423-0x10015c173510000, quorum=127.0.0.1:62362, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-11T16:27:09,907 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-12-11T16:27:09,909 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43423-0x10015c173510000, quorum=127.0.0.1:62362, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-11T16:27:09,920 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-11T16:27:09,932 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42551-0x10015c173510001, quorum=127.0.0.1:62362, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-11T16:27:09,932 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43423-0x10015c173510000, quorum=127.0.0.1:62362, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-11T16:27:09,932 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42551-0x10015c173510001, quorum=127.0.0.1:62362, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:27:09,932 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43423-0x10015c173510000, quorum=127.0.0.1:62362, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:27:09,933 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=7446a65196a1,43423,1733934429385, sessionid=0x10015c173510000, setting cluster-up flag (Was=false)
2024-12-11T16:27:09,948 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43423-0x10015c173510000, quorum=127.0.0.1:62362, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:27:09,948 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42551-0x10015c173510001, quorum=127.0.0.1:62362, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:27:09,973 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-11T16:27:09,974 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=7446a65196a1,43423,1733934429385
2024-12-11T16:27:09,990 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43423-0x10015c173510000, quorum=127.0.0.1:62362, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:27:09,990 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42551-0x10015c173510001, quorum=127.0.0.1:62362, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:27:10,015 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-11T16:27:10,016 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=7446a65196a1,43423,1733934429385
2024-12-11T16:27:10,017 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:39461/user/jenkins/test-data/4e5db2d3-08f7-3881-3540-ed81e84fdce7/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-12-11T16:27:10,018 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-12-11T16:27:10,019 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-12-11T16:27:10,019 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-11T16:27:10,019 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 7446a65196a1,43423,1733934429385
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-11T16:27:10,020 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/7446a65196a1:0, corePoolSize=5, maxPoolSize=5
2024-12-11T16:27:10,020 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/7446a65196a1:0, corePoolSize=5, maxPoolSize=5
2024-12-11T16:27:10,020 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/7446a65196a1:0, corePoolSize=5, maxPoolSize=5
2024-12-11T16:27:10,020 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/7446a65196a1:0, corePoolSize=5, maxPoolSize=5
2024-12-11T16:27:10,020 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/7446a65196a1:0, corePoolSize=10, maxPoolSize=10
2024-12-11T16:27:10,020 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:27:10,020 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/7446a65196a1:0, corePoolSize=2, maxPoolSize=2
2024-12-11T16:27:10,020 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:27:10,023 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733934460023
2024-12-11T16:27:10,024 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-11T16:27:10,024 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-11T16:27:10,024 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-11T16:27:10,024 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-11T16:27:10,024 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-11T16:27:10,024 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-11T16:27:10,024 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-11T16:27:10,024 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-11T16:27:10,024 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-11T16:27:10,024 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-12-11T16:27:10,024 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-11T16:27:10,024 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-11T16:27:10,025 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-11T16:27:10,025 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-11T16:27:10,025 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/7446a65196a1:0:becomeActiveMaster-HFileCleaner.large.0-1733934430025,5,FailOnTimeoutGroup]
2024-12-11T16:27:10,025 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/7446a65196a1:0:becomeActiveMaster-HFileCleaner.small.0-1733934430025,5,FailOnTimeoutGroup]
2024-12-11T16:27:10,025 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-11T16:27:10,025 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-11T16:27:10,025 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-11T16:27:10,025 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-11T16:27:10,025 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:27:10,025 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-11T16:27:10,030 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36765 is added to blk_1073741831_1007 (size=1321)
2024-12-11T16:27:10,030 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43057 is added to blk_1073741831_1007 (size=1321)
2024-12-11T16:27:10,031 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:39461/user/jenkins/test-data/4e5db2d3-08f7-3881-3540-ed81e84fdce7/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-12-11T16:27:10,032 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:39461/user/jenkins/test-data/4e5db2d3-08f7-3881-3540-ed81e84fdce7
2024-12-11T16:27:10,037 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43057 is added to blk_1073741832_1008 (size=32)
2024-12-11T16:27:10,037 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36765 is added to blk_1073741832_1008 (size=32)
2024-12-11T16:27:10,038 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T16:27:10,039 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-11T16:27:10,040 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-11T16:27:10,040 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:27:10,040 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T16:27:10,040 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-11T16:27:10,041 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-11T16:27:10,042 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:27:10,042 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T16:27:10,042 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-11T16:27:10,043 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-11T16:27:10,043 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:27:10,044 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T16:27:10,044 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-11T16:27:10,045 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-11T16:27:10,045 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:27:10,045 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T16:27:10,045 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-11T16:27:10,046 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39461/user/jenkins/test-data/4e5db2d3-08f7-3881-3540-ed81e84fdce7/data/hbase/meta/1588230740
2024-12-11T16:27:10,046 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39461/user/jenkins/test-data/4e5db2d3-08f7-3881-3540-ed81e84fdce7/data/hbase/meta/1588230740
2024-12-11T16:27:10,047 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-11T16:27:10,047 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-11T16:27:10,048 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-11T16:27:10,049 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-11T16:27:10,051 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39461/user/jenkins/test-data/4e5db2d3-08f7-3881-3540-ed81e84fdce7/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-11T16:27:10,051 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=859801, jitterRate=0.09329444169998169}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-11T16:27:10,052 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1733934430038Initializing all the Stores at 1733934430038Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733934430038Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733934430039 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733934430039Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733934430039Cleaning up temporary data from old regions at 1733934430047 (+8 ms)Region opened successfully at 1733934430052 (+5 ms)
2024-12-11T16:27:10,052 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-11T16:27:10,052 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-11T16:27:10,052 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-11T16:27:10,052 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-11T16:27:10,052 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-11T16:27:10,053 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-11T16:27:10,053 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733934430052Disabling compacts and flushes for region at 1733934430052Disabling writes for close at 1733934430052Writing region close event to WAL at 1733934430053 (+1 ms)Closed at 1733934430053
2024-12-11T16:27:10,054 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-11T16:27:10,054 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-12-11T16:27:10,054 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-11T16:27:10,055 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-11T16:27:10,056 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-11T16:27:10,063 INFO  [RS:0;7446a65196a1:42551 {}] regionserver.HRegionServer(746): ClusterId : 3127422d-d9e7-4b79-bb99-dc40b8a7b862
2024-12-11T16:27:10,063 DEBUG [RS:0;7446a65196a1:42551 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-11T16:27:10,074 DEBUG [RS:0;7446a65196a1:42551 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-11T16:27:10,074 DEBUG [RS:0;7446a65196a1:42551 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-11T16:27:10,082 DEBUG [RS:0;7446a65196a1:42551 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-11T16:27:10,083 DEBUG [RS:0;7446a65196a1:42551 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@586bad08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7446a65196a1/172.17.0.2:0
2024-12-11T16:27:10,099 DEBUG [RS:0;7446a65196a1:42551 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;7446a65196a1:42551
2024-12-11T16:27:10,099 INFO  [RS:0;7446a65196a1:42551 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-12-11T16:27:10,099 INFO  [RS:0;7446a65196a1:42551 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-12-11T16:27:10,099 DEBUG [RS:0;7446a65196a1:42551 {}] regionserver.HRegionServer(832): About to register with Master.
2024-12-11T16:27:10,100 INFO  [RS:0;7446a65196a1:42551 {}] regionserver.HRegionServer(2659): reportForDuty to master=7446a65196a1,43423,1733934429385 with port=42551, startcode=1733934429538
2024-12-11T16:27:10,100 DEBUG [RS:0;7446a65196a1:42551 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-11T16:27:10,102 INFO  [HMaster-EventLoopGroup-16-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55435, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.7 (auth:SIMPLE), service=RegionServerStatusService
2024-12-11T16:27:10,102 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43423 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 7446a65196a1,42551,1733934429538
2024-12-11T16:27:10,103 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43423 {}] master.ServerManager(517): Registering regionserver=7446a65196a1,42551,1733934429538
2024-12-11T16:27:10,104 DEBUG [RS:0;7446a65196a1:42551 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:39461/user/jenkins/test-data/4e5db2d3-08f7-3881-3540-ed81e84fdce7
2024-12-11T16:27:10,104 DEBUG [RS:0;7446a65196a1:42551 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:39461
2024-12-11T16:27:10,104 DEBUG [RS:0;7446a65196a1:42551 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-12-11T16:27:10,115 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43423-0x10015c173510000, quorum=127.0.0.1:62362, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-11T16:27:10,115 DEBUG [RS:0;7446a65196a1:42551 {}] zookeeper.ZKUtil(111): regionserver:42551-0x10015c173510001, quorum=127.0.0.1:62362, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/7446a65196a1,42551,1733934429538
2024-12-11T16:27:10,115 WARN  [RS:0;7446a65196a1:42551 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-11T16:27:10,116 INFO  [RS:0;7446a65196a1:42551 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-11T16:27:10,116 DEBUG [RS:0;7446a65196a1:42551 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:39461/user/jenkins/test-data/4e5db2d3-08f7-3881-3540-ed81e84fdce7/WALs/7446a65196a1,42551,1733934429538
2024-12-11T16:27:10,116 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [7446a65196a1,42551,1733934429538]
2024-12-11T16:27:10,118 INFO  [RS:0;7446a65196a1:42551 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-11T16:27:10,120 INFO  [RS:0;7446a65196a1:42551 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-11T16:27:10,120 INFO  [RS:0;7446a65196a1:42551 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-11T16:27:10,120 INFO  [RS:0;7446a65196a1:42551 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-11T16:27:10,120 INFO  [RS:0;7446a65196a1:42551 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-12-11T16:27:10,121 INFO  [RS:0;7446a65196a1:42551 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-12-11T16:27:10,121 INFO  [RS:0;7446a65196a1:42551 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-11T16:27:10,121 DEBUG [RS:0;7446a65196a1:42551 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:27:10,121 DEBUG [RS:0;7446a65196a1:42551 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:27:10,121 DEBUG [RS:0;7446a65196a1:42551 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:27:10,121 DEBUG [RS:0;7446a65196a1:42551 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:27:10,121 DEBUG [RS:0;7446a65196a1:42551 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:27:10,121 DEBUG [RS:0;7446a65196a1:42551 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/7446a65196a1:0, corePoolSize=2, maxPoolSize=2
2024-12-11T16:27:10,121 DEBUG [RS:0;7446a65196a1:42551 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:27:10,121 DEBUG [RS:0;7446a65196a1:42551 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:27:10,121 DEBUG [RS:0;7446a65196a1:42551 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:27:10,121 DEBUG [RS:0;7446a65196a1:42551 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:27:10,121 DEBUG [RS:0;7446a65196a1:42551 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:27:10,121 DEBUG [RS:0;7446a65196a1:42551 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/7446a65196a1:0, corePoolSize=1, maxPoolSize=1
2024-12-11T16:27:10,121 DEBUG [RS:0;7446a65196a1:42551 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/7446a65196a1:0, corePoolSize=3, maxPoolSize=3
2024-12-11T16:27:10,121 DEBUG [RS:0;7446a65196a1:42551 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/7446a65196a1:0, corePoolSize=3, maxPoolSize=3
2024-12-11T16:27:10,122 INFO  [RS:0;7446a65196a1:42551 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-11T16:27:10,122 INFO  [RS:0;7446a65196a1:42551 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-11T16:27:10,122 INFO  [RS:0;7446a65196a1:42551 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-11T16:27:10,122 INFO  [RS:0;7446a65196a1:42551 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-11T16:27:10,122 INFO  [RS:0;7446a65196a1:42551 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-11T16:27:10,122 INFO  [RS:0;7446a65196a1:42551 {}] hbase.ChoreService(168): Chore ScheduledChore name=7446a65196a1,42551,1733934429538-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-11T16:27:10,133 INFO  [RS:0;7446a65196a1:42551 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-11T16:27:10,134 INFO  [RS:0;7446a65196a1:42551 {}] hbase.ChoreService(168): Chore ScheduledChore name=7446a65196a1,42551,1733934429538-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-11T16:27:10,134 INFO  [RS:0;7446a65196a1:42551 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T16:27:10,134 INFO  [RS:0;7446a65196a1:42551 {}] regionserver.Replication(171): 7446a65196a1,42551,1733934429538 started
2024-12-11T16:27:10,146 INFO  [RS:0;7446a65196a1:42551 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T16:27:10,146 INFO  [RS:0;7446a65196a1:42551 {}] regionserver.HRegionServer(1482): Serving as 7446a65196a1,42551,1733934429538, RpcServer on 7446a65196a1/172.17.0.2:42551, sessionid=0x10015c173510001
2024-12-11T16:27:10,146 DEBUG [RS:0;7446a65196a1:42551 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-11T16:27:10,146 DEBUG [RS:0;7446a65196a1:42551 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 7446a65196a1,42551,1733934429538
2024-12-11T16:27:10,146 DEBUG [RS:0;7446a65196a1:42551 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7446a65196a1,42551,1733934429538'
2024-12-11T16:27:10,146 DEBUG [RS:0;7446a65196a1:42551 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-11T16:27:10,146 DEBUG [RS:0;7446a65196a1:42551 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-11T16:27:10,147 DEBUG [RS:0;7446a65196a1:42551 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-11T16:27:10,147 DEBUG [RS:0;7446a65196a1:42551 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-11T16:27:10,147 DEBUG [RS:0;7446a65196a1:42551 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 7446a65196a1,42551,1733934429538
2024-12-11T16:27:10,147 DEBUG [RS:0;7446a65196a1:42551 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7446a65196a1,42551,1733934429538'
2024-12-11T16:27:10,147 DEBUG [RS:0;7446a65196a1:42551 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-11T16:27:10,147 DEBUG [RS:0;7446a65196a1:42551 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-11T16:27:10,147 DEBUG [RS:0;7446a65196a1:42551 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-11T16:27:10,147 INFO  [RS:0;7446a65196a1:42551 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-11T16:27:10,147 INFO  [RS:0;7446a65196a1:42551 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-11T16:27:10,206 WARN  [7446a65196a1:43423 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-12-11T16:27:10,249 INFO  [RS:0;7446a65196a1:42551 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7446a65196a1%2C42551%2C1733934429538, suffix=, logDir=hdfs://localhost:39461/user/jenkins/test-data/4e5db2d3-08f7-3881-3540-ed81e84fdce7/WALs/7446a65196a1,42551,1733934429538, archiveDir=hdfs://localhost:39461/user/jenkins/test-data/4e5db2d3-08f7-3881-3540-ed81e84fdce7/oldWALs, maxLogs=32
2024-12-11T16:27:10,249 INFO  [RS:0;7446a65196a1:42551 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7446a65196a1%2C42551%2C1733934429538.1733934430249
2024-12-11T16:27:10,254 INFO  [RS:0;7446a65196a1:42551 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4e5db2d3-08f7-3881-3540-ed81e84fdce7/WALs/7446a65196a1,42551,1733934429538/7446a65196a1%2C42551%2C1733934429538.1733934430249
2024-12-11T16:27:10,255 DEBUG [RS:0;7446a65196a1:42551 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42477:42477),(127.0.0.1/127.0.0.1:40365:40365)]
2024-12-11T16:27:10,428 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:27:10,428 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:27:10,456 DEBUG [7446a65196a1:43423 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-12-11T16:27:10,457 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=7446a65196a1,42551,1733934429538
2024-12-11T16:27:10,458 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7446a65196a1,42551,1733934429538, state=OPENING
2024-12-11T16:27:10,479 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-11T16:27:10,490 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43423-0x10015c173510000, quorum=127.0.0.1:62362, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:27:10,490 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42551-0x10015c173510001, quorum=127.0.0.1:62362, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:27:10,490 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-11T16:27:10,490 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-11T16:27:10,490 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-11T16:27:10,490 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=7446a65196a1,42551,1733934429538}]
2024-12-11T16:27:10,643 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-11T16:27:10,645 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-17-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47195, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-11T16:27:10,648 INFO  [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-12-11T16:27:10,648 INFO  [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-11T16:27:10,649 INFO  [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7446a65196a1%2C42551%2C1733934429538.meta, suffix=.meta, logDir=hdfs://localhost:39461/user/jenkins/test-data/4e5db2d3-08f7-3881-3540-ed81e84fdce7/WALs/7446a65196a1,42551,1733934429538, archiveDir=hdfs://localhost:39461/user/jenkins/test-data/4e5db2d3-08f7-3881-3540-ed81e84fdce7/oldWALs, maxLogs=32
2024-12-11T16:27:10,650 INFO  [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 7446a65196a1%2C42551%2C1733934429538.meta.1733934430649.meta
2024-12-11T16:27:10,654 INFO  [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4e5db2d3-08f7-3881-3540-ed81e84fdce7/WALs/7446a65196a1,42551,1733934429538/7446a65196a1%2C42551%2C1733934429538.meta.1733934430649.meta
2024-12-11T16:27:10,655 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40365:40365),(127.0.0.1/127.0.0.1:42477:42477)]
2024-12-11T16:27:10,655 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-11T16:27:10,655 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-11T16:27:10,655 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-11T16:27:10,656 INFO  [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-11T16:27:10,656 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-11T16:27:10,656 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T16:27:10,656 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-12-11T16:27:10,656 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-12-11T16:27:10,657 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-11T16:27:10,657 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-11T16:27:10,657 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:27:10,658 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T16:27:10,658 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-11T16:27:10,658 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-11T16:27:10,658 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:27:10,659 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T16:27:10,659 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-11T16:27:10,659 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-11T16:27:10,659 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:27:10,660 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T16:27:10,660 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-11T16:27:10,661 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-11T16:27:10,661 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T16:27:10,661 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T16:27:10,661 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-11T16:27:10,662 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39461/user/jenkins/test-data/4e5db2d3-08f7-3881-3540-ed81e84fdce7/data/hbase/meta/1588230740
2024-12-11T16:27:10,663 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39461/user/jenkins/test-data/4e5db2d3-08f7-3881-3540-ed81e84fdce7/data/hbase/meta/1588230740
2024-12-11T16:27:10,664 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-11T16:27:10,664 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-11T16:27:10,664 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-11T16:27:10,665 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-11T16:27:10,666 INFO  [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=711916, jitterRate=-0.09475237131118774}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-11T16:27:10,666 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-12-11T16:27:10,666 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1733934430656Writing region info on filesystem at 1733934430656Initializing all the Stores at 1733934430656Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733934430657 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733934430657Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733934430657Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733934430657Cleaning up temporary data from old regions at 1733934430664 (+7 ms)Running coprocessor post-open hooks at 1733934430666 (+2 ms)Region opened successfully at 1733934430666
2024-12-11T16:27:10,667 INFO  [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733934430643
2024-12-11T16:27:10,669 DEBUG [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-11T16:27:10,669 INFO  [RS_OPEN_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-12-11T16:27:10,669 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=7446a65196a1,42551,1733934429538
2024-12-11T16:27:10,670 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7446a65196a1,42551,1733934429538, state=OPEN
2024-12-11T16:27:10,720 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43423-0x10015c173510000, quorum=127.0.0.1:62362, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-11T16:27:10,720 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42551-0x10015c173510001, quorum=127.0.0.1:62362, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-11T16:27:10,720 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=7446a65196a1,42551,1733934429538
2024-12-11T16:27:10,720 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-11T16:27:10,720 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-11T16:27:10,722 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-12-11T16:27:10,722 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7446a65196a1,42551,1733934429538 in 230 msec
2024-12-11T16:27:10,724 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-12-11T16:27:10,724 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 668 msec
2024-12-11T16:27:10,724 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-11T16:27:10,724 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-12-11T16:27:10,725 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-11T16:27:10,725 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7446a65196a1,42551,1733934429538, seqNum=-1]
2024-12-11T16:27:10,726 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-11T16:27:10,727 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-17-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33187, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-11T16:27:10,731 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 713 msec
2024-12-11T16:27:10,731 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733934430731, completionTime=-1
2024-12-11T16:27:10,732 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-12-11T16:27:10,732 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-12-11T16:27:10,733 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-12-11T16:27:10,733 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733934490733
2024-12-11T16:27:10,733 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733934550733
2024-12-11T16:27:10,733 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 1 msec
2024-12-11T16:27:10,734 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7446a65196a1,43423,1733934429385-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-11T16:27:10,734 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7446a65196a1,43423,1733934429385-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T16:27:10,734 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7446a65196a1,43423,1733934429385-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T16:27:10,734 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-7446a65196a1:43423, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T16:27:10,734 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-11T16:27:10,734 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-12-11T16:27:10,736 DEBUG [master/7446a65196a1:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-12-11T16:27:10,737 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.089sec
2024-12-11T16:27:10,737 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-11T16:27:10,737 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-11T16:27:10,737 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-11T16:27:10,737 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-11T16:27:10,737 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-11T16:27:10,737 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7446a65196a1,43423,1733934429385-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-11T16:27:10,737 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7446a65196a1,43423,1733934429385-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-11T16:27:10,740 DEBUG [master/7446a65196a1:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-12-11T16:27:10,740 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-11T16:27:10,740 INFO  [master/7446a65196a1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7446a65196a1,43423,1733934429385-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T16:27:10,763 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e3560f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-11T16:27:10,763 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7446a65196a1,43423,-1 for getting cluster id
2024-12-11T16:27:10,763 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-12-11T16:27:10,764 DEBUG [HMaster-EventLoopGroup-16-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3127422d-d9e7-4b79-bb99-dc40b8a7b862'
2024-12-11T16:27:10,764 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-12-11T16:27:10,764 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3127422d-d9e7-4b79-bb99-dc40b8a7b862"

2024-12-11T16:27:10,765 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7dbeee3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-11T16:27:10,765 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7446a65196a1,43423,-1]
2024-12-11T16:27:10,765 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-12-11T16:27:10,765 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T16:27:10,766 INFO  [HMaster-EventLoopGroup-16-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54770, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-12-11T16:27:10,767 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5262831f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-11T16:27:10,767 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-11T16:27:10,768 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7446a65196a1,42551,1733934429538, seqNum=-1]
2024-12-11T16:27:10,768 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-11T16:27:10,769 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-17-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39676, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-11T16:27:10,770 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=7446a65196a1,43423,1733934429385
2024-12-11T16:27:10,771 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T16:27:10,773 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-12-11T16:27:10,773 INFO  [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-11T16:27:10,775 INFO  [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=test.com%2C8080%2C1, suffix=, logDir=hdfs://localhost:39461/user/jenkins/test-data/4e5db2d3-08f7-3881-3540-ed81e84fdce7/WALs/test.com,8080,1, archiveDir=hdfs://localhost:39461/user/jenkins/test-data/4e5db2d3-08f7-3881-3540-ed81e84fdce7/oldWALs, maxLogs=32
2024-12-11T16:27:10,776 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor test.com%2C8080%2C1.1733934430776
2024-12-11T16:27:10,781 INFO  [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4e5db2d3-08f7-3881-3540-ed81e84fdce7/WALs/test.com,8080,1/test.com%2C8080%2C1.1733934430776
2024-12-11T16:27:10,782 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40365:40365),(127.0.0.1/127.0.0.1:42477:42477)]
2024-12-11T16:27:10,782 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor test.com%2C8080%2C1.1733934430782
2024-12-11T16:27:10,787 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:27:10,787 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:27:10,787 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:27:10,787 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:27:10,787 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:27:10,787 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/4e5db2d3-08f7-3881-3540-ed81e84fdce7/WALs/test.com,8080,1/test.com%2C8080%2C1.1733934430776 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/4e5db2d3-08f7-3881-3540-ed81e84fdce7/WALs/test.com,8080,1/test.com%2C8080%2C1.1733934430782
2024-12-11T16:27:10,788 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42477:42477),(127.0.0.1/127.0.0.1:40365:40365)]
2024-12-11T16:27:10,788 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:39461/user/jenkins/test-data/4e5db2d3-08f7-3881-3540-ed81e84fdce7/WALs/test.com,8080,1/test.com%2C8080%2C1.1733934430776 is not closed yet, will try archiving it next time
2024-12-11T16:27:10,789 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:27:10,790 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36765 is added to blk_1073741835_1011 (size=93)
2024-12-11T16:27:10,790 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43057 is added to blk_1073741835_1011 (size=93)
2024-12-11T16:27:10,790 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:27:10,790 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:27:10,790 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:27:10,790 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:27:10,792 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43057 is added to blk_1073741836_1012 (size=93)
2024-12-11T16:27:10,792 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36765 is added to blk_1073741836_1012 (size=93)
2024-12-11T16:27:11,191 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:39461/user/jenkins/test-data/4e5db2d3-08f7-3881-3540-ed81e84fdce7/WALs/test.com,8080,1/test.com%2C8080%2C1.1733934430776 to hdfs://localhost:39461/user/jenkins/test-data/4e5db2d3-08f7-3881-3540-ed81e84fdce7/oldWALs/test.com%2C8080%2C1.1733934430776
2024-12-11T16:27:11,193 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/4e5db2d3-08f7-3881-3540-ed81e84fdce7/oldWALs
2024-12-11T16:27:11,193 INFO  [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog test.com%2C8080%2C1:(num 1733934430782)
2024-12-11T16:27:11,193 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-12-11T16:27:11,193 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-11T16:27:11,193 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-11T16:27:11,193 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T16:27:11,193 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T16:27:11,193 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-12-11T16:27:11,193 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-12-11T16:27:11,194 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1276771852, stopped=false
2024-12-11T16:27:11,194 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=7446a65196a1,43423,1733934429385
2024-12-11T16:27:11,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43423-0x10015c173510000, quorum=127.0.0.1:62362, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-11T16:27:11,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42551-0x10015c173510001, quorum=127.0.0.1:62362, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-11T16:27:11,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43423-0x10015c173510000, quorum=127.0.0.1:62362, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:27:11,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42551-0x10015c173510001, quorum=127.0.0.1:62362, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:27:11,215 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-11T16:27:11,215 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-11T16:27:11,215 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-11T16:27:11,215 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T16:27:11,216 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '7446a65196a1,42551,1733934429538' *****
2024-12-11T16:27:11,216 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-12-11T16:27:11,216 INFO  [RS:0;7446a65196a1:42551 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-11T16:27:11,216 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42551-0x10015c173510001, quorum=127.0.0.1:62362, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-11T16:27:11,216 INFO  [RS:0;7446a65196a1:42551 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-11T16:27:11,216 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-12-11T16:27:11,216 INFO  [RS:0;7446a65196a1:42551 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-11T16:27:11,216 INFO  [RS:0;7446a65196a1:42551 {}] regionserver.HRegionServer(959): stopping server 7446a65196a1,42551,1733934429538
2024-12-11T16:27:11,216 INFO  [RS:0;7446a65196a1:42551 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-11T16:27:11,216 INFO  [RS:0;7446a65196a1:42551 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;7446a65196a1:42551.
2024-12-11T16:27:11,216 DEBUG [RS:0;7446a65196a1:42551 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-11T16:27:11,216 DEBUG [RS:0;7446a65196a1:42551 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T16:27:11,216 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43423-0x10015c173510000, quorum=127.0.0.1:62362, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-11T16:27:11,216 INFO  [RS:0;7446a65196a1:42551 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-11T16:27:11,217 INFO  [RS:0;7446a65196a1:42551 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-11T16:27:11,217 INFO  [RS:0;7446a65196a1:42551 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-11T16:27:11,217 INFO  [RS:0;7446a65196a1:42551 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-12-11T16:27:11,217 INFO  [RS:0;7446a65196a1:42551 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close
2024-12-11T16:27:11,217 DEBUG [RS:0;7446a65196a1:42551 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740}
2024-12-11T16:27:11,217 DEBUG [RS:0;7446a65196a1:42551 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-12-11T16:27:11,217 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-11T16:27:11,217 INFO  [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-11T16:27:11,217 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-11T16:27:11,218 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 1 ms
2024-12-11T16:27:11,218 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-11T16:27:11,218 INFO  [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=74 B heapSize=1.22 KB
2024-12-11T16:27:11,241 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39461/user/jenkins/test-data/4e5db2d3-08f7-3881-3540-ed81e84fdce7/data/hbase/meta/1588230740/.tmp/ns/463fba1270784148abb26d0e8e77264f is 43, key is default/ns:d/1733934430727/Put/seqid=0
2024-12-11T16:27:11,251 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36765 is added to blk_1073741837_1013 (size=5153)
2024-12-11T16:27:11,251 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43057 is added to blk_1073741837_1013 (size=5153)
2024-12-11T16:27:11,417 DEBUG [RS:0;7446a65196a1:42551 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-12-11T16:27:11,429 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:27:11,429 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:27:11,617 DEBUG [RS:0;7446a65196a1:42551 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-12-11T16:27:11,650 INFO  [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:39461/user/jenkins/test-data/4e5db2d3-08f7-3881-3540-ed81e84fdce7/data/hbase/meta/1588230740/.tmp/ns/463fba1270784148abb26d0e8e77264f
2024-12-11T16:27:11,655 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39461/user/jenkins/test-data/4e5db2d3-08f7-3881-3540-ed81e84fdce7/data/hbase/meta/1588230740/.tmp/ns/463fba1270784148abb26d0e8e77264f as hdfs://localhost:39461/user/jenkins/test-data/4e5db2d3-08f7-3881-3540-ed81e84fdce7/data/hbase/meta/1588230740/ns/463fba1270784148abb26d0e8e77264f
2024-12-11T16:27:11,661 INFO  [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39461/user/jenkins/test-data/4e5db2d3-08f7-3881-3540-ed81e84fdce7/data/hbase/meta/1588230740/ns/463fba1270784148abb26d0e8e77264f, entries=2, sequenceid=6, filesize=5.0 K
2024-12-11T16:27:11,662 INFO  [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 444ms, sequenceid=6, compaction requested=false
2024-12-11T16:27:11,662 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-12-11T16:27:11,666 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39461/user/jenkins/test-data/4e5db2d3-08f7-3881-3540-ed81e84fdce7/data/hbase/meta/1588230740/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1
2024-12-11T16:27:11,667 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-11T16:27:11,667 INFO  [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-11T16:27:11,667 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733934431217Running coprocessor pre-close hooks at 1733934431217Disabling compacts and flushes for region at 1733934431217Disabling writes for close at 1733934431218 (+1 ms)Obtaining lock to block concurrent updates at 1733934431218Preparing flush snapshotting stores in 1588230740 at 1733934431218Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=74, getHeapSize=1184, getOffHeapSize=0, getCellsCount=2 at 1733934431218Flushing stores of hbase:meta,,1.1588230740 at 1733934431219 (+1 ms)Flushing 1588230740/ns: creating writer at 1733934431219Flushing 1588230740/ns: appending metadata at 1733934431240 (+21 ms)Flushing 1588230740/ns: closing flushed file at 1733934431240Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6d88a434: reopening flushed file at 1733934431655 (+415 ms)Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 444ms, sequenceid=6, compaction requested=false at 1733934431662 (+7 ms)Writing region close event to WAL at 1733934431663 (+1 ms)Running coprocessor post-close hooks at 1733934431666 (+3 ms)Closed at 1733934431667 (+1 ms)
2024-12-11T16:27:11,667 DEBUG [RS_CLOSE_META-regionserver/7446a65196a1:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-12-11T16:27:11,770 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-11T16:27:11,771 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:27:11,771 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:27:11,771 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:27:11,772 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:27:11,772 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:27:11,772 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:27:11,773 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:27:11,773 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:27:11,773 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:27:11,773 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:27:11,805 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:27:11,806 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:27:11,806 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:27:11,806 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:27:11,807 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:27:11,807 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:27:11,810 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:27:11,811 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:27:11,811 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:27:11,814 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T16:27:11,818 INFO  [RS:0;7446a65196a1:42551 {}] regionserver.HRegionServer(976): stopping server 7446a65196a1,42551,1733934429538; all regions closed.
2024-12-11T16:27:11,818 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:27:11,818 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:27:11,818 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:27:11,823 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:27:11,824 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:27:11,827 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43057 is added to blk_1073741834_1010 (size=1152)
2024-12-11T16:27:11,827 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36765 is added to blk_1073741834_1010 (size=1152)
2024-12-11T16:27:11,833 DEBUG [RS:0;7446a65196a1:42551 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/4e5db2d3-08f7-3881-3540-ed81e84fdce7/oldWALs
2024-12-11T16:27:11,833 INFO  [RS:0;7446a65196a1:42551 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 7446a65196a1%2C42551%2C1733934429538.meta:.meta(num 1733934430649)
2024-12-11T16:27:11,833 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:27:11,833 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:27:11,833 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:27:11,833 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:27:11,833 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:27:11,839 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43057 is added to blk_1073741833_1009 (size=93)
2024-12-11T16:27:11,839 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36765 is added to blk_1073741833_1009 (size=93)
2024-12-11T16:27:12,122 INFO  [regionserver/7446a65196a1:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-11T16:27:12,183 INFO  [regionserver/7446a65196a1:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-12-11T16:27:12,183 INFO  [regionserver/7446a65196a1:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-12-11T16:27:12,241 DEBUG [RS:0;7446a65196a1:42551 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/4e5db2d3-08f7-3881-3540-ed81e84fdce7/oldWALs
2024-12-11T16:27:12,241 INFO  [RS:0;7446a65196a1:42551 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 7446a65196a1%2C42551%2C1733934429538:(num 1733934430249)
2024-12-11T16:27:12,241 DEBUG [RS:0;7446a65196a1:42551 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T16:27:12,241 INFO  [RS:0;7446a65196a1:42551 {}] regionserver.LeaseManager(133): Closed leases
2024-12-11T16:27:12,241 INFO  [RS:0;7446a65196a1:42551 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-11T16:27:12,241 INFO  [RS:0;7446a65196a1:42551 {}] hbase.ChoreService(370): Chore service for: regionserver/7446a65196a1:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-12-11T16:27:12,241 INFO  [RS:0;7446a65196a1:42551 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-11T16:27:12,241 INFO  [regionserver/7446a65196a1:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-11T16:27:12,241 INFO  [RS:0;7446a65196a1:42551 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42551
2024-12-11T16:27:12,273 INFO  [RS:0;7446a65196a1:42551 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-11T16:27:12,273 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42551-0x10015c173510001, quorum=127.0.0.1:62362, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/7446a65196a1,42551,1733934429538
2024-12-11T16:27:12,273 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43423-0x10015c173510000, quorum=127.0.0.1:62362, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-11T16:27:12,290 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [7446a65196a1,42551,1733934429538]
2024-12-11T16:27:12,298 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/7446a65196a1,42551,1733934429538 already deleted, retry=false
2024-12-11T16:27:12,298 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 7446a65196a1,42551,1733934429538 expired; onlineServers=0
2024-12-11T16:27:12,298 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '7446a65196a1,43423,1733934429385' *****
2024-12-11T16:27:12,298 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-12-11T16:27:12,298 INFO  [M:0;7446a65196a1:43423 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-11T16:27:12,298 INFO  [M:0;7446a65196a1:43423 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-11T16:27:12,298 DEBUG [M:0;7446a65196a1:43423 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-12-11T16:27:12,298 DEBUG [M:0;7446a65196a1:43423 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-12-11T16:27:12,298 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-12-11T16:27:12,298 DEBUG [master/7446a65196a1:0:becomeActiveMaster-HFileCleaner.small.0-1733934430025 {}] cleaner.HFileCleaner(306): Exit Thread[master/7446a65196a1:0:becomeActiveMaster-HFileCleaner.small.0-1733934430025,5,FailOnTimeoutGroup]
2024-12-11T16:27:12,299 INFO  [M:0;7446a65196a1:43423 {}] hbase.ChoreService(370): Chore service for: master/7446a65196a1:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-12-11T16:27:12,299 INFO  [M:0;7446a65196a1:43423 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-11T16:27:12,299 DEBUG [M:0;7446a65196a1:43423 {}] master.HMaster(1795): Stopping service threads
2024-12-11T16:27:12,299 INFO  [M:0;7446a65196a1:43423 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-12-11T16:27:12,299 INFO  [M:0;7446a65196a1:43423 {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-11T16:27:12,299 INFO  [M:0;7446a65196a1:43423 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-12-11T16:27:12,299 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-12-11T16:27:12,299 DEBUG [master/7446a65196a1:0:becomeActiveMaster-HFileCleaner.large.0-1733934430025 {}] cleaner.HFileCleaner(306): Exit Thread[master/7446a65196a1:0:becomeActiveMaster-HFileCleaner.large.0-1733934430025,5,FailOnTimeoutGroup]
2024-12-11T16:27:12,306 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43423-0x10015c173510000, quorum=127.0.0.1:62362, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-12-11T16:27:12,306 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43423-0x10015c173510000, quorum=127.0.0.1:62362, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T16:27:12,307 DEBUG [M:0;7446a65196a1:43423 {}] zookeeper.ZKUtil(347): master:43423-0x10015c173510000, quorum=127.0.0.1:62362, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-12-11T16:27:12,307 WARN  [M:0;7446a65196a1:43423 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-12-11T16:27:12,307 INFO  [M:0;7446a65196a1:43423 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:39461/user/jenkins/test-data/4e5db2d3-08f7-3881-3540-ed81e84fdce7/.lastflushedseqids
2024-12-11T16:27:12,318 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43057 is added to blk_1073741838_1014 (size=99)
2024-12-11T16:27:12,318 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36765 is added to blk_1073741838_1014 (size=99)
2024-12-11T16:27:12,319 INFO  [M:0;7446a65196a1:43423 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-12-11T16:27:12,319 INFO  [M:0;7446a65196a1:43423 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-12-11T16:27:12,319 DEBUG [M:0;7446a65196a1:43423 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-11T16:27:12,319 INFO  [M:0;7446a65196a1:43423 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T16:27:12,319 DEBUG [M:0;7446a65196a1:43423 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T16:27:12,319 DEBUG [M:0;7446a65196a1:43423 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-11T16:27:12,319 DEBUG [M:0;7446a65196a1:43423 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T16:27:12,319 INFO  [M:0;7446a65196a1:43423 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=7.67 KB heapSize=11.34 KB
2024-12-11T16:27:12,341 DEBUG [M:0;7446a65196a1:43423 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39461/user/jenkins/test-data/4e5db2d3-08f7-3881-3540-ed81e84fdce7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2622557a92ec45b7b931b80352235b6e is 82, key is hbase:meta,,1/info:regioninfo/1733934430669/Put/seqid=0
2024-12-11T16:27:12,359 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36765 is added to blk_1073741839_1015 (size=5672)
2024-12-11T16:27:12,360 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43057 is added to blk_1073741839_1015 (size=5672)
2024-12-11T16:27:12,360 INFO  [M:0;7446a65196a1:43423 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:39461/user/jenkins/test-data/4e5db2d3-08f7-3881-3540-ed81e84fdce7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2622557a92ec45b7b931b80352235b6e
2024-12-11T16:27:12,389 DEBUG [M:0;7446a65196a1:43423 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39461/user/jenkins/test-data/4e5db2d3-08f7-3881-3540-ed81e84fdce7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d726c3b8b8ad468d82446edbd6581502 is 240, key is \x00\x00\x00\x00\x00\x00\x00\x01/proc:d/1733934430731/Put/seqid=0
2024-12-11T16:27:12,390 INFO  [RS:0;7446a65196a1:42551 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-11T16:27:12,390 INFO  [RS:0;7446a65196a1:42551 {}] regionserver.HRegionServer(1031): Exiting; stopping=7446a65196a1,42551,1733934429538; zookeeper connection closed.
2024-12-11T16:27:12,391 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4300142 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4300142
2024-12-11T16:27:12,391 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42551-0x10015c173510001, quorum=127.0.0.1:62362, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-11T16:27:12,391 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42551-0x10015c173510001, quorum=127.0.0.1:62362, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-11T16:27:12,391 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-12-11T16:27:12,411 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43057 is added to blk_1073741840_1016 (size=5275)
2024-12-11T16:27:12,411 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36765 is added to blk_1073741840_1016 (size=5275)
2024-12-11T16:27:12,429 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,42595,1733934228675/7446a65196a1%2C42595%2C1733934228675.1733934229064
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:27:12,429 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34525/user/jenkins/test-data/280b219a-62be-a66b-33f9-9232d2e1b8ee/WALs/7446a65196a1,45947,1733934227248/7446a65196a1%2C45947%2C1733934227248.meta.1733934228399.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T16:27:12,812 INFO  [M:0;7446a65196a1:43423 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.06 KB at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:39461/user/jenkins/test-data/4e5db2d3-08f7-3881-3540-ed81e84fdce7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d726c3b8b8ad468d82446edbd6581502
2024-12-11T16:27:12,837 DEBUG [M:0;7446a65196a1:43423 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39461/user/jenkins/test-data/4e5db2d3-08f7-3881-3540-ed81e84fdce7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5df41dee02ea41da9bc09a0da7909bf7 is 69, key is 7446a65196a1,42551,1733934429538/rs:state/1733934430103/Put/seqid=0
2024-12-11T16:27:12,841 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43057 is added to blk_1073741841_1017 (size=5156)
2024-12-11T16:27:12,841 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36765 is added to blk_1073741841_1017 (size=5156)
2024-12-11T16:27:12,842 INFO  [M:0;7446a65196a1:43423 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:39461/user/jenkins/test-data/4e5db2d3-08f7-3881-3540-ed81e84fdce7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5df41dee02ea41da9bc09a0da7909bf7
2024-12-11T16:27:12,867 DEBUG [M:0;7446a65196a1:43423 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39461/user/jenkins/test-data/4e5db2d3-08f7-3881-3540-ed81e84fdce7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/e607fadaa2864e19842c3f16f8a99be5 is 52, key is load_balancer_on/state:d/1733934430772/Put/seqid=0
2024-12-11T16:27:12,872 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43057 is added to blk_1073741842_1018 (size=5056)
2024-12-11T16:27:12,872 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36765 is added to blk_1073741842_1018 (size=5056)
2024-12-11T16:27:12,872 INFO  [M:0;7446a65196a1:43423 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:39461/user/jenkins/test-data/4e5db2d3-08f7-3881-3540-ed81e84fdce7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/e607fadaa2864e19842c3f16f8a99be5
2024-12-11T16:27:12,877 DEBUG [M:0;7446a65196a1:43423 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39461/user/jenkins/test-data/4e5db2d3-08f7-3881-3540-ed81e84fdce7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2622557a92ec45b7b931b80352235b6e as hdfs://localhost:39461/user/jenkins/test-data/4e5db2d3-08f7-3881-3540-ed81e84fdce7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2622557a92ec45b7b931b80352235b6e
2024-12-11T16:27:12,881 INFO  [M:0;7446a65196a1:43423 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39461/user/jenkins/test-data/4e5db2d3-08f7-3881-3540-ed81e84fdce7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2622557a92ec45b7b931b80352235b6e, entries=8, sequenceid=29, filesize=5.5 K
2024-12-11T16:27:12,883 DEBUG [M:0;7446a65196a1:43423 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39461/user/jenkins/test-data/4e5db2d3-08f7-3881-3540-ed81e84fdce7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d726c3b8b8ad468d82446edbd6581502 as hdfs://localhost:39461/user/jenkins/test-data/4e5db2d3-08f7-3881-3540-ed81e84fdce7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/d726c3b8b8ad468d82446edbd6581502
2024-12-11T16:27:12,888 INFO  [M:0;7446a65196a1:43423 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39461/user/jenkins/test-data/4e5db2d3-08f7-3881-3540-ed81e84fdce7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/d726c3b8b8ad468d82446edbd6581502, entries=3, sequenceid=29, filesize=5.2 K
2024-12-11T16:27:12,889 DEBUG [M:0;7446a65196a1:43423 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39461/user/jenkins/test-data/4e5db2d3-08f7-3881-3540-ed81e84fdce7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5df41dee02ea41da9bc09a0da7909bf7 as hdfs://localhost:39461/user/jenkins/test-data/4e5db2d3-08f7-3881-3540-ed81e84fdce7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/5df41dee02ea41da9bc09a0da7909bf7
2024-12-11T16:27:12,897 INFO  [M:0;7446a65196a1:43423 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39461/user/jenkins/test-data/4e5db2d3-08f7-3881-3540-ed81e84fdce7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/5df41dee02ea41da9bc09a0da7909bf7, entries=1, sequenceid=29, filesize=5.0 K
2024-12-11T16:27:12,899 DEBUG [M:0;7446a65196a1:43423 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39461/user/jenkins/test-data/4e5db2d3-08f7-3881-3540-ed81e84fdce7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/e607fadaa2864e19842c3f16f8a99be5 as hdfs://localhost:39461/user/jenkins/test-data/4e5db2d3-08f7-3881-3540-ed81e84fdce7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/e607fadaa2864e19842c3f16f8a99be5
2024-12-11T16:27:12,905 INFO  [M:0;7446a65196a1:43423 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39461/user/jenkins/test-data/4e5db2d3-08f7-3881-3540-ed81e84fdce7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/e607fadaa2864e19842c3f16f8a99be5, entries=1, sequenceid=29, filesize=4.9 K
2024-12-11T16:27:12,906 INFO  [M:0;7446a65196a1:43423 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 587ms, sequenceid=29, compaction requested=false
2024-12-11T16:27:12,907 INFO  [M:0;7446a65196a1:43423 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T16:27:12,907 DEBUG [M:0;7446a65196a1:43423 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733934432319Disabling compacts and flushes for region at 1733934432319Disabling writes for close at 1733934432319Obtaining lock to block concurrent updates at 1733934432319Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733934432319Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=7850, getHeapSize=11544, getOffHeapSize=0, getCellsCount=36 at 1733934432320 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733934432320Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733934432320Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733934432340 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733934432340Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733934432368 (+28 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733934432389 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733934432389Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733934432817 (+428 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733934432837 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733934432837Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733934432847 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733934432867 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733934432867Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5a58fc17: reopening flushed file at 1733934432876 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@d78a997: reopening flushed file at 1733934432882 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3d08469d: reopening flushed file at 1733934432888 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@462e0bd6: reopening flushed file at 1733934432897 (+9 ms)Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 587ms, sequenceid=29, compaction requested=false at 1733934432906 (+9 ms)Writing region close event to WAL at 1733934432907 (+1 ms)Closed at 1733934432907
2024-12-11T16:27:12,911 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:27:12,911 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:27:12,911 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:27:12,911 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:27:12,911 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T16:27:12,913 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43057 is added to blk_1073741830_1006 (size=10311)
2024-12-11T16:27:12,914 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36765 is added to blk_1073741830_1006 (size=10311)
2024-12-11T16:27:12,914 INFO  [M:0;7446a65196a1:43423 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-12-11T16:27:12,914 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-11T16:27:12,914 INFO  [M:0;7446a65196a1:43423 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43423
2024-12-11T16:27:12,914 INFO  [M:0;7446a65196a1:43423 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-11T16:27:13,063 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43423-0x10015c173510000, quorum=127.0.0.1:62362, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-11T16:27:13,063 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43423-0x10015c173510000, quorum=127.0.0.1:62362, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-11T16:27:13,063 INFO  [M:0;7446a65196a1:43423 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-11T16:27:13,067 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@46b56cc{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T16:27:13,067 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@fbadfdb{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-11T16:27:13,067 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-11T16:27:13,067 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6650dd0b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-11T16:27:13,068 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5154bbd2{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81c691dc-3bb4-2c8e-f7a7-649598fcf539/hadoop.log.dir/,STOPPED}
2024-12-11T16:27:13,079 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-11T16:27:13,079 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-11T16:27:13,079 WARN  [BP-313031944-172.17.0.2-1733934427590 heartbeating to localhost/127.0.0.1:39461 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-11T16:27:13,079 WARN  [BP-313031944-172.17.0.2-1733934427590 heartbeating to localhost/127.0.0.1:39461 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-313031944-172.17.0.2-1733934427590 (Datanode Uuid 09d0c22a-7611-4458-883f-f262917054ee) service to localhost/127.0.0.1:39461
2024-12-11T16:27:13,080 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81c691dc-3bb4-2c8e-f7a7-649598fcf539/cluster_7a27fac0-c771-36a1-1040-de522363619d/data/data3/current/BP-313031944-172.17.0.2-1733934427590 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T16:27:13,080 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81c691dc-3bb4-2c8e-f7a7-649598fcf539/cluster_7a27fac0-c771-36a1-1040-de522363619d/data/data4/current/BP-313031944-172.17.0.2-1733934427590 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T16:27:13,080 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-11T16:27:13,088 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4981e138{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T16:27:13,088 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1f07cd2f{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-11T16:27:13,088 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-11T16:27:13,088 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3b90357{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-11T16:27:13,088 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5ebcbe6a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81c691dc-3bb4-2c8e-f7a7-649598fcf539/hadoop.log.dir/,STOPPED}
2024-12-11T16:27:13,093 WARN  [BP-313031944-172.17.0.2-1733934427590 heartbeating to localhost/127.0.0.1:39461 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-11T16:27:13,093 WARN  [BP-313031944-172.17.0.2-1733934427590 heartbeating to localhost/127.0.0.1:39461 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-313031944-172.17.0.2-1733934427590 (Datanode Uuid 6b7dba04-6238-40ee-8627-a43ae0cf1e09) service to localhost/127.0.0.1:39461
2024-12-11T16:27:13,094 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81c691dc-3bb4-2c8e-f7a7-649598fcf539/cluster_7a27fac0-c771-36a1-1040-de522363619d/data/data2/current/BP-313031944-172.17.0.2-1733934427590 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T16:27:13,095 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81c691dc-3bb4-2c8e-f7a7-649598fcf539/cluster_7a27fac0-c771-36a1-1040-de522363619d/data/data1/current/BP-313031944-172.17.0.2-1733934427590 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T16:27:13,095 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-11T16:27:13,095 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-11T16:27:13,096 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-11T16:27:13,105 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@443fd1e4{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-11T16:27:13,106 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@bc56624{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-11T16:27:13,106 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-11T16:27:13,106 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@79a533ce{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-11T16:27:13,107 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@417d4fc2{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81c691dc-3bb4-2c8e-f7a7-649598fcf539/hadoop.log.dir/,STOPPED}
2024-12-11T16:27:13,134 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-12-11T16:27:13,148 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-12-11T16:27:13,157 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnNothingWritten Thread=273 (was 235)
Potentially hanging thread: LeaseRenewer:jenkins@localhost:39461
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:39461
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-16-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-45-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-44-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-43-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (548703531) connection to localhost/127.0.0.1:39461 from jenkins.hfs.7
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: HMaster-EventLoopGroup-16-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-45-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.7@localhost:39461
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:39461
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (548703531) connection to localhost/127.0.0.1:39461 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:39461
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-44-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (548703531) connection to localhost/127.0.0.1:39461 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-42-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-16-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-6
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: nioEventLoopGroup-44-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-43-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-42-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-45-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-42-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-43-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=532 (was 515) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=380 (was 353) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2258 (was 1375) - AvailableMemoryMB LEAK? -